|
|
[110][111]
|
[116][117]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:04
|
2026/09/03 16:10:05
|
|
1 second 113 ms
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 121
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
f4ab84f4-63f7-4ffd-9e57-1bc858c28878
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_f4ab84f4-63f7-4ffd-9e57-1bc858c28878
|
|
|
|
|
[112]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:05
|
2026/09/03 16:10:05
|
|
268 ms
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 123
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
9fcf03d6-e3f1-4cf0-ba94-f9fec4989937
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_9fcf03d6-e3f1-4cf0-ba94-f9fec4989937
|
|
|
|
|
[113]
|
[118]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:05
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 125
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
30c3a325-39ef-4d92-aeb2-0c3cb47450ee
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_30c3a325-39ef-4d92-aeb2-0c3cb47450ee
|
|
|
|
|
[116]
|
[123][124]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 131
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
bf40aa6a-2502-42af-8fda-de65e57da0fc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_bf40aa6a-2502-42af-8fda-de65e57da0fc
|
|
|
|
|
[118]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 135
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
7023641f-3361-4da3-9af4-ecef5cacf90a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_7023641f-3361-4da3-9af4-ecef5cacf90a
|
|
|
|
|
[121]
|
[128][129]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 139
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
e9888ae1-7199-4cff-9c64-58021fbdf24e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_e9888ae1-7199-4cff-9c64-58021fbdf24e
|
|
|
|
|
[122]
|
[130][131]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 141
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
0736c6a9-8500-4c2b-a735-a0089e70f6bc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_0736c6a9-8500-4c2b-a735-a0089e70f6bc
|
|
|
|
|
[123][124]
|
[132][133]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 143
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
c01a3fb1-bdc7-4ec1-a598-e4f540be20fe
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_c01a3fb1-bdc7-4ec1-a598-e4f540be20fe
|
|
|
|
|
[125]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 145
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
79715b50-a0c6-4e7e-989e-0d369605fa7e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_79715b50-a0c6-4e7e-989e-0d369605fa7e
|
|
|
|
|
[127]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
2026/09/03 16:10:07
|
|
127 ms
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 149
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
7865d33d-b14d-45dd-80a0-07c104be9353
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_7865d33d-b14d-45dd-80a0-07c104be9353
|
|
|
|
|
[128][129]
|
[136][137]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 151
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
7addeaeb-55db-4d99-a6af-39b0ebebf3da
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_7addeaeb-55db-4d99-a6af-39b0ebebf3da
|
|
|
|
|
[130]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 153
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
2630ec1c-1b31-4c50-a65b-55f5decbd2fc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_2630ec1c-1b31-4c50-a65b-55f5decbd2fc
|
|
|
|
|
[131]
|
[138][139]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:08
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 155
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
4909aecf-283f-47b5-8b1a-eaca18732b21
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4909aecf-283f-47b5-8b1a-eaca18732b21
|
|
|
|
|
[133]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:08
|
2026/09/03 16:10:08
|
|
266 ms
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 159
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
f8e84450-8ff3-4bd6-ac2b-82af9ebbfca0
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_f8e84450-8ff3-4bd6-ac2b-82af9ebbfca0
|
|
|
|
|
[134]
|
[141][142]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:08
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 161
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
e9f70b88-ec0d-429a-ae12-eef385da88ec
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_e9f70b88-ec0d-429a-ae12-eef385da88ec
|
|
|
|
|
[135][136]
|
[143][144]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:08
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 163
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
c2fa0c35-4ecf-4024-9fd5-3129c9236878
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_c2fa0c35-4ecf-4024-9fd5-3129c9236878
|
|
|
|
|
[137][139]
|
[145][146]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:09
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 165
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
2f228dd1-be15-416d-be3d-8dc0abcdbd80
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_2f228dd1-be15-416d-be3d-8dc0abcdbd80
|
|
|
|
|
[142]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:09
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 173
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
531ff59e-678b-4e2c-8bde-a0f5c5f9248b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_531ff59e-678b-4e2c-8bde-a0f5c5f9248b
|
|
|
|
|
[143]
|
[149]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:09
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 175
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
3bb4164c-8dcf-4a51-bc6d-b9c29174b839
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_3bb4164c-8dcf-4a51-bc6d-b9c29174b839
|
|
|
|
|
[146]
|
[150][151]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:10
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 177
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
4f641188-f012-4bf6-bce6-417e8736f9c1
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4f641188-f012-4bf6-bce6-417e8736f9c1
|
|
|
|
|
[145][148]
|
[152][153]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:10
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 179
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
4a4d2cea-6df1-4258-81eb-757d681d47f8
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4a4d2cea-6df1-4258-81eb-757d681d47f8
|
|
|
|
|
[147]
|
[154]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:10
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 181
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
3f39aeed-ff3c-4900-8789-60d0940d1a76
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_3f39aeed-ff3c-4900-8789-60d0940d1a76
|
|
|
|
|
[151][152]
|
[157][158]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:17
|
2026/09/03 16:10:23
|
|
5 seconds 520 ms
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 185
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
a1541722-9f42-44d1-b623-e01e03ca7219
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_a1541722-9f42-44d1-b623-e01e03ca7219
|
|
|
|
|
[153][154]
|
[159][160]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:23
|
2026/09/03 16:10:26
|
|
3 seconds 768 ms
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 187
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
ce16240e-152e-4179-aeb2-45a9b7a0bbbf
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_ce16240e-152e-4179-aeb2-45a9b7a0bbbf
|
|
|
|
|
[155][156][157]
|
[161][162][163]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:26
|
2026/09/03 16:10:39
|
2026/09/03 16:17:54
|
12 seconds 670 ms
|
7 minutes 27 seconds
|
write_operation {
input {
common {
plan_id: 189
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
0d73ec80-efc3-457f-b7a0-ecf03792de7e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_0d73ec80-efc3-457f-b7a0-ecf03792de7e
|
|
|
|
|
[158][159][160]
|
[164][165][166]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:07
|
2026/09/03 16:12:38
|
2026/09/03 16:12:38
|
31 seconds 764 ms
|
31 seconds 769 ms
|
write_operation {
input {
common {
plan_id: 0
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
72c531e6-b91e-430c-a90f-107332e492cb
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_72c531e6-b91e-430c-a90f-107332e492cb
|
|
|
|
|
[164][165]
|
[170][171]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:39
|
2026/09/03 16:13:06
|
2026/09/03 16:13:06
|
27 seconds 620 ms
|
27 seconds 625 ms
|
write_operation {
input {
common {
plan_id: 8
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
4a0e6fa4-11f8-45c5-9138-12142a2c4d93
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_4a0e6fa4-11f8-45c5-9138-12142a2c4d93
|
|
|
|
|
[100][101][97][98][99]
|
[105][106][107][108][109]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:03:17
|
2026/09/03 16:03:44
|
2026/09/03 16:03:44
|
26 seconds 948 ms
|
26 seconds 976 ms
|
common {
plan_id: 112
}
limit {
input {
common {
plan_id: 111
}
read {
data_source {
format: "delta"
schema: ""
paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
}
}
}
limit: 1000
}
|
CLOSED
|
5c4d2c10-fdcb-48fb-b90d-06cbd97c3009
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_5c4d2c10-fdcb-48fb-b90d-06cbd97c3009
|
|
|
|
|
[219][220]
|
[225][226]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
26 seconds 854 ms
|
26 seconds 857 ms
|
write_operation {
input {
common {
plan_id: 114
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
21211c3d-a004-4101-a1bb-488c89fddc2f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_21211c3d-a004-4101-a1bb-488c89fddc2f
|
|
|
|
|
[169][170]
|
[175][176]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:07
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
25 seconds 362 ms
|
25 seconds 366 ms
|
write_operation {
input {
common {
plan_id: 16
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
035259f2-8534-4df9-9603-193cb3db6fe8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_035259f2-8534-4df9-9603-193cb3db6fe8
|
|
|
|
|
[106][107]
|
[112][113]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:09:49
|
2026/09/03 16:10:00
|
2026/09/03 16:10:00
|
10 seconds 248 ms
|
10 seconds 252 ms
|
write_operation {
input {
common {
plan_id: 117
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
99accdac-7c3e-412d-adf9-fcb58b5c916c
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_99accdac-7c3e-412d-adf9-fcb58b5c916c
|
|
|
|
|
[149][150]
|
[155][156]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:10
|
2026/09/03 16:10:17
|
2026/09/03 16:10:17
|
7 seconds 162 ms
|
7 seconds 163 ms
|
write_operation {
input {
common {
plan_id: 183
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
19e2d57c-1fb5-4d37-a377-c4460bc08d30
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_19e2d57c-1fb5-4d37-a377-c4460bc08d30
|
|
|
|
|
[108][109]
|
[114][115]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:00
|
2026/09/03 16:10:04
|
2026/09/03 16:10:04
|
4 seconds 83 ms
|
4 seconds 85 ms
|
write_operation {
input {
common {
plan_id: 119
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
4834ae02-5360-4c58-8f85-4f9f8f34dae7
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4834ae02-5360-4c58-8f85-4f9f8f34dae7
|
|
|
|
|
[102][103]
|
[110]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:03:59
|
2026/09/03 16:04:01
|
2026/09/03 16:04:01
|
2 seconds 133 ms
|
2 seconds 277 ms
|
common {
plan_id: 114
}
aggregate {
input {
common {
plan_id: 113
}
read {
data_source {
format: "delta"
schema: ""
paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
}
}
}
group_type: GROUP_TYPE_GROUPBY
aggregate_expressions {
unresolved_function {
function_name: "count"
arguments {
literal {
integer: 1
}
}
}
}
}
|
CLOSED
|
7e420069-8d5c-4a04-8ffd-b5541331883b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_7e420069-8d5c-4a04-8ffd-b5541331883b
|
|
|
|
|
[104][105]
|
[111]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:04:22
|
2026/09/03 16:04:23
|
2026/09/03 16:04:23
|
1 second 293 ms
|
1 second 306 ms
|
common {
plan_id: 116
}
aggregate {
input {
common {
plan_id: 115
}
read {
data_source {
format: "delta"
schema: ""
paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
}
}
}
group_type: GROUP_TYPE_GROUPBY
aggregate_expressions {
unresolved_function {
function_name: "count"
arguments {
literal {
integer: 1
}
}
}
}
}
|
CLOSED
|
1534209b-e69e-4668-879a-4f7b18f0c8ed
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_1534209b-e69e-4668-879a-4f7b18f0c8ed
|
|
|
|
|
|
[99]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:56:11
|
2026/09/03 15:56:11
|
2026/09/03 15:56:11
|
776 ms
|
779 ms
|
sql_command {
input {
common {
plan_id: 102
}
sql {
query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n VendorID long,\n tpep_pickup_datetime timestamp,\n tpep_dropoff_datetime timestamp,\n passenger_count long,\n trip_distance double,\n RatecodeID long,\n store_and_fwd_flag string,\n PULocationID long,\n DOLocationID long,\n payment_type long,\n fare_amount double,\n extra double,\n mta_tax double,\n tip_amount double,\n tolls_amount double,\n improvement_surcharge double,\n total_amount double,\n congestion_surcharge double,\n airport_fee double,\n load_timestamp timestamp,\n year int,\n month int,\n day int\n)\nusing delta\npartitioned by (year,month)\n"
}
}
}
|
CLOSED
|
9fcede98-796d-4cc6-b745-53573fede9fd
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_9fcede98-796d-4cc6-b745-53573fede9fd
|
|
DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION
+details
INTERNAL: [DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION] Cannot create table ('`yellow_taxi`.`trips`'). The associated location ('file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips') is not empty and also not a Delta table.
|
|
|
[138]
|
[147][148]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:09
|
2026/09/03 16:10:09
|
2026/09/03 16:10:09
|
560 ms
|
567 ms
|
write_operation {
input {
common {
plan_id: 167
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
0e0ae871-5c31-43fe-bfcd-3145d557e602
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_0e0ae871-5c31-43fe-bfcd-3145d557e602
|
|
|
|
|
|
[97]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:59
|
2026/09/03 15:55:59
|
2026/09/03 15:55:59
|
510 ms
|
515 ms
|
sql_command {
input {
common {
plan_id: 98
}
sql {
query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n VendorID long,\n tpep_pickup_datetime timestamp,\n tpep_dropoff_datetime timestamp,\n passenger_count long,\n trip_distance double,\n RatecodeID long,\n store_and_fwd_flag string,\n PULocationID long,\n DOLocationID long,\n payment_type long,\n fare_amount double,\n extra double,\n mta_tax double,\n tip_amount double,\n tolls_amount double,\n improvement_surcharge double,\n total_amount double,\n congestion_surcharge double,\n airport_fee double,\n load_timestamp timestamp,\n year int,\n month int,\n day int\n)\nusing delta\npartitioned by (year,month)\n"
}
}
}
|
CLOSED
|
a13d02e2-eec6-48b6-814e-ba08e461b69e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_a13d02e2-eec6-48b6-814e-ba08e461b69e
|
|
SCHEMA_NOT_FOUND
+details
INTERNAL: [SCHEMA_NOT_FOUND] The schema `spark_catalog`.`yellow_taxi` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.
To tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704
|
|
|
[168]
|
[174]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
499 ms
|
503 ms
|
write_operation {
input {
common {
plan_id: 14
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d2e00546-45c8-4a1f-890f-63c0315ed1cc
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d2e00546-45c8-4a1f-890f-63c0315ed1cc
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[117][119]
|
[125][126]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
2026/09/03 16:10:06
|
2026/09/03 16:10:06
|
390 ms
|
390 ms
|
write_operation {
input {
common {
plan_id: 133
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
3353fded-646f-4e2b-94ee-ddf45ec66e18
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_3353fded-646f-4e2b-94ee-ddf45ec66e18
|
|
|
|
|
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:56:10
|
2026/09/03 15:56:10
|
2026/09/03 15:56:10
|
357 ms
|
378 ms
|
common {
plan_id: 101
}
limit {
input {
common {
plan_id: 100
}
local_relation {
data: "\377\377\377\377@\000\000\000[truncated(size=152)]"
}
}
limit: 1000
}
|
CLOSED
|
2fb37124-09dd-46af-a3ea-7f3ff3899f9a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_2fb37124-09dd-46af-a3ea-7f3ff3899f9a
|
|
|
|
|
[126]
|
[134][135]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
2026/09/03 16:10:07
|
2026/09/03 16:10:07
|
333 ms
|
334 ms
|
write_operation {
input {
common {
plan_id: 147
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
14b3700d-9902-44c3-8c7b-7d1ab2be7dac
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_14b3700d-9902-44c3-8c7b-7d1ab2be7dac
|
|
|
|
|
[114]
|
[119][120]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:05
|
2026/09/03 16:10:06
|
2026/09/03 16:10:06
|
322 ms
|
322 ms
|
write_operation {
input {
common {
plan_id: 127
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
24da78ab-53f3-47e8-b928-e057f6c960f7
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_24da78ab-53f3-47e8-b928-e057f6c960f7
|
|
|
|
|
[167]
|
[173]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
304 ms
|
310 ms
|
write_operation {
input {
common {
plan_id: 12
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
ccc38014-821f-42c5-b1f5-65d5700fbadf
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_ccc38014-821f-42c5-b1f5-65d5700fbadf
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[115]
|
[121][122]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:05
|
2026/09/03 16:10:06
|
2026/09/03 16:10:06
|
289 ms
|
289 ms
|
write_operation {
input {
common {
plan_id: 129
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
6780f38f-c162-42bf-98a5-81e0579a82ee
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_6780f38f-c162-42bf-98a5-81e0579a82ee
|
|
|
|
|
[192]
|
[198]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
279 ms
|
284 ms
|
write_operation {
input {
common {
plan_id: 60
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2c86dd1a-1570-4f98-9e2e-9b81a80d2006
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2c86dd1a-1570-4f98-9e2e-9b81a80d2006
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[179]
|
[185]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
250 ms
|
255 ms
|
write_operation {
input {
common {
plan_id: 34
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
58782597-3c4f-4429-82f1-d7be3d74c34e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_58782597-3c4f-4429-82f1-d7be3d74c34e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[245]
|
[251]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
230 ms
|
236 ms
|
write_operation {
input {
common {
plan_id: 164
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
68456b2c-116e-405c-9494-0c45481a30f9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_68456b2c-116e-405c-9494-0c45481a30f9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
|
[98]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:56:09
|
2026/09/03 15:56:10
|
2026/09/03 15:56:10
|
216 ms
|
235 ms
|
sql_command {
input {
common {
plan_id: 99
}
sql {
query: "create schema if not exists spark_catalog.yellow_taxi"
}
}
}
|
CLOSED
|
3322b461-013d-4354-aa6f-30d2254e748c
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_3322b461-013d-4354-aa6f-30d2254e748c
|
|
|
|
|
[233]
|
[239]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
209 ms
|
212 ms
|
write_operation {
input {
common {
plan_id: 140
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
8d6ace22-b5ab-4313-a159-d5f59b0f7b07
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_8d6ace22-b5ab-4313-a159-d5f59b0f7b07
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[85]
|
[85]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
199 ms
|
203 ms
|
sql_command {
input {
common {
plan_id: 86
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-02.parquet`\n "
}
}
}
|
CLOSED
|
300ea7b2-0a1f-4e0b-aefa-ca3b63ffbc78
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_300ea7b2-0a1f-4e0b-aefa-ca3b63ffbc78
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3728L as bigint) AS VendorID#3749L, try_cast(tpep_pickup_datetime#3729 as timestamp) AS tpep_pickup_datetime#3750, try_cast(tpep_dropoff_datetime#3730 as timestamp) AS tpep_dropoff_datetime#3751, try_cast(passenger_count#3731L as bigint) AS passenger_count#3752L, try_cast(trip_distance#3732 as double) AS trip_distance#3753, try_cast(RatecodeID#3733L as bigint) AS RatecodeID#3754L, try_cast(store_and_fwd_flag#3734 as string) AS store_and_fwd_flag#3755, try_cast(PULocationID#3735L as bigint) AS PULocationID#3756L, try_cast(DOLocationID#3736L as bigint) AS DOLocationID#3757L, try_cast(payment_type#3737L as bigint) AS payment_type#3758L, try_cast(fare_amount#3738 as double) AS fare_amount#3759, try_cast(extra#3739 as double) AS extra#3760, try_cast(mta_tax#3740 as double) AS mta_tax#3761, try_cast(tip_amount#3741 as double) AS tip_amount#3762, try_cast(tolls_amount#3742 as double) AS tolls_amount#3763, try_cast(improvement_surcharge#3743 as double) AS improvement_surcharge#3764, try_cast(total_amount#3744 as double) AS total_amount#3765, try_cast(congestion_surcharge#3745 as double) AS congestion_surcharge#3766, try_cast(airport_fee#3746 as double) AS airport_fee#3767, current_timestamp() AS current_timestamp()#3748, coalesce(year(cast(tpep_pickup_datetime#3729 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3768, coalesce(month(cast(tpep_pickup_datetime#3729 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3769, coalesce(day(cast(tpep_pickup_d...
|
|
|
[76]
|
[76]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
191 ms
|
200 ms
|
sql_command {
input {
common {
plan_id: 77
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-10.parquet`\n "
}
}
}
|
CLOSED
|
666d7732-56bb-4a3f-9910-563a42d240b6
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_666d7732-56bb-4a3f-9910-563a42d240b6
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3331L as bigint) AS VendorID#3352L, try_cast(tpep_pickup_datetime#3332 as timestamp) AS tpep_pickup_datetime#3353, try_cast(tpep_dropoff_datetime#3333 as timestamp) AS tpep_dropoff_datetime#3354, try_cast(passenger_count#3334 as bigint) AS passenger_count#3355L, try_cast(trip_distance#3335 as double) AS trip_distance#3356, try_cast(RatecodeID#3336 as bigint) AS RatecodeID#3357L, try_cast(store_and_fwd_flag#3337 as string) AS store_and_fwd_flag#3358, try_cast(PULocationID#3338L as bigint) AS PULocationID#3359L, try_cast(DOLocationID#3339L as bigint) AS DOLocationID#3360L, try_cast(payment_type#3340L as bigint) AS payment_type#3361L, try_cast(fare_amount#3341 as double) AS fare_amount#3362, try_cast(extra#3342 as double) AS extra#3363, try_cast(mta_tax#3343 as double) AS mta_tax#3364, try_cast(tip_amount#3344 as double) AS tip_amount#3365, try_cast(tolls_amount#3345 as double) AS tolls_amount#3366, try_cast(improvement_surcharge#3346 as double) AS improvement_surcharge#3367, try_cast(total_amount#3347 as double) AS total_amount#3368, try_cast(congestion_surcharge#3348 as double) AS congestion_surcharge#3369, try_cast(airport_fee#3349 as double) AS airport_fee#3370, current_timestamp() AS current_timestamp()#3351, coalesce(year(cast(tpep_pickup_datetime#3332 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3371, coalesce(month(cast(tpep_pickup_datetime#3332 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3372, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[132]
|
[140]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:08
|
2026/09/03 16:10:08
|
2026/09/03 16:10:08
|
198 ms
|
199 ms
|
write_operation {
input {
common {
plan_id: 157
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
5d5f4566-bcab-49ad-b1f3-d0255fe910bc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_5d5f4566-bcab-49ad-b1f3-d0255fe910bc
|
|
|
|
|
[258]
|
[264]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
193 ms
|
196 ms
|
write_operation {
input {
common {
plan_id: 190
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f51353ee-b876-4820-8de9-8e595ad43048
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f51353ee-b876-4820-8de9-8e595ad43048
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[72]
|
[72]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
190 ms
|
194 ms
|
sql_command {
input {
common {
plan_id: 73
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-06.parquet`\n "
}
}
}
|
CLOSED
|
e5874da6-8807-4984-a2af-2198993d3c6d
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_e5874da6-8807-4984-a2af-2198993d3c6d
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3155 as bigint) AS VendorID#3176L, try_cast(tpep_pickup_datetime#3156 as timestamp) AS tpep_pickup_datetime#3177, try_cast(tpep_dropoff_datetime#3157 as timestamp) AS tpep_dropoff_datetime#3178, try_cast(passenger_count#3158L as bigint) AS passenger_count#3179L, try_cast(trip_distance#3159 as double) AS trip_distance#3180, try_cast(RatecodeID#3160L as bigint) AS RatecodeID#3181L, try_cast(store_and_fwd_flag#3161 as string) AS store_and_fwd_flag#3182, try_cast(PULocationID#3162 as bigint) AS PULocationID#3183L, try_cast(DOLocationID#3163 as bigint) AS DOLocationID#3184L, try_cast(payment_type#3164L as bigint) AS payment_type#3185L, try_cast(fare_amount#3165 as double) AS fare_amount#3186, try_cast(extra#3166 as double) AS extra#3187, try_cast(mta_tax#3167 as double) AS mta_tax#3188, try_cast(tip_amount#3168 as double) AS tip_amount#3189, try_cast(tolls_amount#3169 as double) AS tolls_amount#3190, try_cast(improvement_surcharge#3170 as double) AS improvement_surcharge#3191, try_cast(total_amount#3171 as double) AS total_amount#3192, try_cast(congestion_surcharge#3172 as double) AS congestion_surcharge#3193, try_cast(airport_fee#3173 as double) AS airport_fee#3194, current_timestamp() AS current_timestamp()#3175, coalesce(year(cast(tpep_pickup_datetime#3156 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3195, coalesce(month(cast(tpep_pickup_datetime#3156 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3196, coalesce(day(cast(tpep_pickup_date...
|
|
|
[96]
|
[96]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:59
|
2026/09/03 15:55:59
|
186 ms
|
192 ms
|
sql_command {
input {
common {
plan_id: 97
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-02.parquet`\n "
}
}
}
|
CLOSED
|
b974847a-e78b-4ca1-9eec-cc51d06f3032
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_b974847a-e78b-4ca1-9eec-cc51d06f3032
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#4213 as bigint) AS VendorID#4234L, try_cast(tpep_pickup_datetime#4214 as timestamp) AS tpep_pickup_datetime#4235, try_cast(tpep_dropoff_datetime#4215 as timestamp) AS tpep_dropoff_datetime#4236, try_cast(passenger_count#4216L as bigint) AS passenger_count#4237L, try_cast(trip_distance#4217 as double) AS trip_distance#4238, try_cast(RatecodeID#4218L as bigint) AS RatecodeID#4239L, try_cast(store_and_fwd_flag#4219 as string) AS store_and_fwd_flag#4240, try_cast(PULocationID#4220 as bigint) AS PULocationID#4241L, try_cast(DOLocationID#4221 as bigint) AS DOLocationID#4242L, try_cast(payment_type#4222L as bigint) AS payment_type#4243L, try_cast(fare_amount#4223 as double) AS fare_amount#4244, try_cast(extra#4224 as double) AS extra#4245, try_cast(mta_tax#4225 as double) AS mta_tax#4246, try_cast(tip_amount#4226 as double) AS tip_amount#4247, try_cast(tolls_amount#4227 as double) AS tolls_amount#4248, try_cast(improvement_surcharge#4228 as double) AS improvement_surcharge#4249, try_cast(total_amount#4229 as double) AS total_amount#4250, try_cast(congestion_surcharge#4230 as double) AS congestion_surcharge#4251, try_cast(airport_fee#4231 as double) AS airport_fee#4252, current_timestamp() AS current_timestamp()#4233, coalesce(year(cast(tpep_pickup_datetime#4214 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4253, coalesce(month(cast(tpep_pickup_datetime#4214 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4254, coalesce(day(cast(tpep_pickup_date...
|
|
|
[94]
|
[94]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
183 ms
|
190 ms
|
sql_command {
input {
common {
plan_id: 95
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-02.parquet`\n "
}
}
}
|
CLOSED
|
2150884e-0543-40e8-9b8e-e5bb1316f685
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_2150884e-0543-40e8-9b8e-e5bb1316f685
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#4125L as bigint) AS VendorID#4146L, try_cast(tpep_pickup_datetime#4126 as timestamp) AS tpep_pickup_datetime#4147, try_cast(tpep_dropoff_datetime#4127 as timestamp) AS tpep_dropoff_datetime#4148, try_cast(passenger_count#4128 as bigint) AS passenger_count#4149L, try_cast(trip_distance#4129 as double) AS trip_distance#4150, try_cast(RatecodeID#4130 as bigint) AS RatecodeID#4151L, try_cast(store_and_fwd_flag#4131 as string) AS store_and_fwd_flag#4152, try_cast(PULocationID#4132L as bigint) AS PULocationID#4153L, try_cast(DOLocationID#4133L as bigint) AS DOLocationID#4154L, try_cast(payment_type#4134L as bigint) AS payment_type#4155L, try_cast(fare_amount#4135 as double) AS fare_amount#4156, try_cast(extra#4136 as double) AS extra#4157, try_cast(mta_tax#4137 as double) AS mta_tax#4158, try_cast(tip_amount#4138 as double) AS tip_amount#4159, try_cast(tolls_amount#4139 as double) AS tolls_amount#4160, try_cast(improvement_surcharge#4140 as double) AS improvement_surcharge#4161, try_cast(total_amount#4141 as double) AS total_amount#4162, try_cast(congestion_surcharge#4142 as double) AS congestion_surcharge#4163, try_cast(airport_fee#4143 as double) AS airport_fee#4164, current_timestamp() AS current_timestamp()#4145, coalesce(year(cast(tpep_pickup_datetime#4126 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4165, coalesce(month(cast(tpep_pickup_datetime#4126 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4166, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[166]
|
[172]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:06
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
176 ms
|
183 ms
|
write_operation {
input {
common {
plan_id: 10
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
95614555-3777-4dfe-826f-dd28d83359e8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_95614555-3777-4dfe-826f-dd28d83359e8
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[140]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:09
|
2026/09/03 16:10:09
|
2026/09/03 16:10:09
|
182 ms
|
182 ms
|
write_operation {
input {
common {
plan_id: 169
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
dcb5e0d2-c2a0-4d54-bde4-c5fe6867a554
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_dcb5e0d2-c2a0-4d54-bde4-c5fe6867a554
|
|
|
|
|
[244]
|
[250]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
169 ms
|
179 ms
|
write_operation {
input {
common {
plan_id: 162
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f988164d-2a30-4c6c-812b-4ad65094c526
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f988164d-2a30-4c6c-812b-4ad65094c526
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[120]
|
[127]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
2026/09/03 16:10:06
|
2026/09/03 16:10:06
|
175 ms
|
176 ms
|
write_operation {
input {
common {
plan_id: 137
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
8b26a669-e3a5-46d2-9682-646315f9fcf6
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_8b26a669-e3a5-46d2-9682-646315f9fcf6
|
|
|
|
|
[209]
|
[215]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
171 ms
|
175 ms
|
write_operation {
input {
common {
plan_id: 94
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
00f8c9d7-c11a-411f-b7be-c0be0e2f29b0
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_00f8c9d7-c11a-411f-b7be-c0be0e2f29b0
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[60]
|
[60]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:53
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
162 ms
|
170 ms
|
sql_command {
input {
common {
plan_id: 61
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-10.parquet`\n "
}
}
}
|
CLOSED
|
94dd56d8-e7d4-4a6c-adab-649ebff775f6
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_94dd56d8-e7d4-4a6c-adab-649ebff775f6
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2627L as bigint) AS VendorID#2648L, try_cast(tpep_pickup_datetime#2628 as timestamp) AS tpep_pickup_datetime#2649, try_cast(tpep_dropoff_datetime#2629 as timestamp) AS tpep_dropoff_datetime#2650, try_cast(passenger_count#2630 as bigint) AS passenger_count#2651L, try_cast(trip_distance#2631 as double) AS trip_distance#2652, try_cast(RatecodeID#2632 as bigint) AS RatecodeID#2653L, try_cast(store_and_fwd_flag#2633 as string) AS store_and_fwd_flag#2654, try_cast(PULocationID#2634L as bigint) AS PULocationID#2655L, try_cast(DOLocationID#2635L as bigint) AS DOLocationID#2656L, try_cast(payment_type#2636L as bigint) AS payment_type#2657L, try_cast(fare_amount#2637 as double) AS fare_amount#2658, try_cast(extra#2638 as double) AS extra#2659, try_cast(mta_tax#2639 as double) AS mta_tax#2660, try_cast(tip_amount#2640 as double) AS tip_amount#2661, try_cast(tolls_amount#2641 as double) AS tolls_amount#2662, try_cast(improvement_surcharge#2642 as double) AS improvement_surcharge#2663, try_cast(total_amount#2643 as double) AS total_amount#2664, try_cast(congestion_surcharge#2644 as double) AS congestion_surcharge#2665, try_cast(airport_fee#2645 as double) AS airport_fee#2666, current_timestamp() AS current_timestamp()#2647, coalesce(year(cast(tpep_pickup_datetime#2628 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2667, coalesce(month(cast(tpep_pickup_datetime#2628 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2668, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[205]
|
[211]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
159 ms
|
170 ms
|
write_operation {
input {
common {
plan_id: 86
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d7ae4bfa-6ae1-4130-a89f-a2a3062ce403
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d7ae4bfa-6ae1-4130-a89f-a2a3062ce403
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[178]
|
[184]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
164 ms
|
169 ms
|
write_operation {
input {
common {
plan_id: 32
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
ed081ed8-4ab9-4e4d-a9d8-f25faeb6c9ed
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_ed081ed8-4ab9-4e4d-a9d8-f25faeb6c9ed
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[222]
|
[228]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
151 ms
|
165 ms
|
write_operation {
input {
common {
plan_id: 118
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
e95632ff-dda1-4fe8-90ec-9a939137983b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_e95632ff-dda1-4fe8-90ec-9a939137983b
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[95]
|
[95]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
156 ms
|
164 ms
|
sql_command {
input {
common {
plan_id: 96
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-05.parquet`\n "
}
}
}
|
CLOSED
|
ddf797a2-7955-4646-aba6-4f9a3ce7fd77
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_ddf797a2-7955-4646-aba6-4f9a3ce7fd77
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#4169 as bigint) AS VendorID#4190L, try_cast(tpep_pickup_datetime#4170 as timestamp) AS tpep_pickup_datetime#4191, try_cast(tpep_dropoff_datetime#4171 as timestamp) AS tpep_dropoff_datetime#4192, try_cast(passenger_count#4172L as bigint) AS passenger_count#4193L, try_cast(trip_distance#4173 as double) AS trip_distance#4194, try_cast(RatecodeID#4174L as bigint) AS RatecodeID#4195L, try_cast(store_and_fwd_flag#4175 as string) AS store_and_fwd_flag#4196, try_cast(PULocationID#4176 as bigint) AS PULocationID#4197L, try_cast(DOLocationID#4177 as bigint) AS DOLocationID#4198L, try_cast(payment_type#4178L as bigint) AS payment_type#4199L, try_cast(fare_amount#4179 as double) AS fare_amount#4200, try_cast(extra#4180 as double) AS extra#4201, try_cast(mta_tax#4181 as double) AS mta_tax#4202, try_cast(tip_amount#4182 as double) AS tip_amount#4203, try_cast(tolls_amount#4183 as double) AS tolls_amount#4204, try_cast(improvement_surcharge#4184 as double) AS improvement_surcharge#4205, try_cast(total_amount#4185 as double) AS total_amount#4206, try_cast(congestion_surcharge#4186 as double) AS congestion_surcharge#4207, try_cast(airport_fee#4187 as double) AS airport_fee#4208, current_timestamp() AS current_timestamp()#4189, coalesce(year(cast(tpep_pickup_datetime#4170 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4209, coalesce(month(cast(tpep_pickup_datetime#4170 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4210, coalesce(day(cast(tpep_pickup_date...
|
|
|
[250]
|
[256]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
158 ms
|
162 ms
|
write_operation {
input {
common {
plan_id: 174
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
12b1ae76-d166-4c27-9c07-8187996ee91e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_12b1ae76-d166-4c27-9c07-8187996ee91e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[232]
|
[238]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
110 ms
|
161 ms
|
write_operation {
input {
common {
plan_id: 138
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
52a96a13-df5e-4a95-af5a-163a6252c863
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_52a96a13-df5e-4a95-af5a-163a6252c863
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[223]
|
[229]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
156 ms
|
160 ms
|
write_operation {
input {
common {
plan_id: 120
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
581465f6-3066-4298-b9b9-5ec1adcbc575
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_581465f6-3066-4298-b9b9-5ec1adcbc575
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[171]
|
[177]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
156 ms
|
159 ms
|
write_operation {
input {
common {
plan_id: 18
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
ce551e70-2c11-40d6-bee9-c12c33e80063
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_ce551e70-2c11-40d6-bee9-c12c33e80063
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[141]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:09
|
2026/09/03 16:10:09
|
2026/09/03 16:10:09
|
156 ms
|
156 ms
|
write_operation {
input {
common {
plan_id: 171
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
46569e48-f731-4f2a-85b8-f216c5e2460e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_46569e48-f731-4f2a-85b8-f216c5e2460e
|
|
|
|
|
[172]
|
[178]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
140 ms
|
151 ms
|
write_operation {
input {
common {
plan_id: 20
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
993baa0e-ddcb-4699-b50a-50e6c4c21fde
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_993baa0e-ddcb-4699-b50a-50e6c4c21fde
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[184]
|
[190]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
143 ms
|
150 ms
|
write_operation {
input {
common {
plan_id: 44
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7a5df2e0-b57e-470b-94e5-badc2b56aa44
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7a5df2e0-b57e-470b-94e5-badc2b56aa44
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[204]
|
[210]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
143 ms
|
148 ms
|
write_operation {
input {
common {
plan_id: 84
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
6bc41597-674a-4fee-9ab3-55277c54abe3
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_6bc41597-674a-4fee-9ab3-55277c54abe3
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[203]
|
[209]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
142 ms
|
148 ms
|
write_operation {
input {
common {
plan_id: 82
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b69fd70b-bb76-4198-91e3-0f7223de0693
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b69fd70b-bb76-4198-91e3-0f7223de0693
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[80]
|
[80]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
133 ms
|
147 ms
|
sql_command {
input {
common {
plan_id: 81
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-11.parquet`\n "
}
}
}
|
CLOSED
|
300aa833-632d-41e2-b4d0-2915f9f05b5a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_300aa833-632d-41e2-b4d0-2915f9f05b5a
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3508L as bigint) AS VendorID#3529L, try_cast(tpep_pickup_datetime#3509 as timestamp) AS tpep_pickup_datetime#3530, try_cast(tpep_dropoff_datetime#3510 as timestamp) AS tpep_dropoff_datetime#3531, try_cast(passenger_count#3511 as bigint) AS passenger_count#3532L, try_cast(trip_distance#3512 as double) AS trip_distance#3533, try_cast(RatecodeID#3513 as bigint) AS RatecodeID#3534L, try_cast(store_and_fwd_flag#3514 as string) AS store_and_fwd_flag#3535, try_cast(PULocationID#3515L as bigint) AS PULocationID#3536L, try_cast(DOLocationID#3516L as bigint) AS DOLocationID#3537L, try_cast(payment_type#3517L as bigint) AS payment_type#3538L, try_cast(fare_amount#3518 as double) AS fare_amount#3539, try_cast(extra#3519 as double) AS extra#3540, try_cast(mta_tax#3520 as double) AS mta_tax#3541, try_cast(tip_amount#3521 as double) AS tip_amount#3542, try_cast(tolls_amount#3522 as double) AS tolls_amount#3543, try_cast(improvement_surcharge#3523 as double) AS improvement_surcharge#3544, try_cast(total_amount#3524 as double) AS total_amount#3545, try_cast(congestion_surcharge#3525 as double) AS congestion_surcharge#3546, try_cast(airport_fee#3526 as double) AS airport_fee#3547, current_timestamp() AS current_timestamp()#3528, coalesce(year(cast(tpep_pickup_datetime#3509 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3548, coalesce(month(cast(tpep_pickup_datetime#3509 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3549, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[256]
|
[262]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
143 ms
|
146 ms
|
write_operation {
input {
common {
plan_id: 186
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
131986c8-8cfc-4db3-a6cc-6201942f62e0
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_131986c8-8cfc-4db3-a6cc-6201942f62e0
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[210]
|
[216]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
141 ms
|
145 ms
|
write_operation {
input {
common {
plan_id: 96
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
3c9481a0-0f2d-428a-9746-fcd72062da9f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_3c9481a0-0f2d-428a-9746-fcd72062da9f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[201]
|
[207]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
140 ms
|
144 ms
|
write_operation {
input {
common {
plan_id: 78
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
be22460e-a3d6-4c1d-98e9-0f8c38a34805
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_be22460e-a3d6-4c1d-98e9-0f8c38a34805
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[182]
|
[188]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
139 ms
|
143 ms
|
write_operation {
input {
common {
plan_id: 40
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a9b43a10-8ee3-4cdc-b188-b967035b5a2e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a9b43a10-8ee3-4cdc-b188-b967035b5a2e
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[64]
|
[64]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
134 ms
|
141 ms
|
sql_command {
input {
common {
plan_id: 65
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-02.parquet`\n "
}
}
}
|
CLOSED
|
1e92a838-5a6e-43a9-acb6-29c21ec6736e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_1e92a838-5a6e-43a9-acb6-29c21ec6736e
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2803 as bigint) AS VendorID#2824L, try_cast(tpep_pickup_datetime#2804 as timestamp) AS tpep_pickup_datetime#2825, try_cast(tpep_dropoff_datetime#2805 as timestamp) AS tpep_dropoff_datetime#2826, try_cast(passenger_count#2806L as bigint) AS passenger_count#2827L, try_cast(trip_distance#2807 as double) AS trip_distance#2828, try_cast(RatecodeID#2808L as bigint) AS RatecodeID#2829L, try_cast(store_and_fwd_flag#2809 as string) AS store_and_fwd_flag#2830, try_cast(PULocationID#2810 as bigint) AS PULocationID#2831L, try_cast(DOLocationID#2811 as bigint) AS DOLocationID#2832L, try_cast(payment_type#2812L as bigint) AS payment_type#2833L, try_cast(fare_amount#2813 as double) AS fare_amount#2834, try_cast(extra#2814 as double) AS extra#2835, try_cast(mta_tax#2815 as double) AS mta_tax#2836, try_cast(tip_amount#2816 as double) AS tip_amount#2837, try_cast(tolls_amount#2817 as double) AS tolls_amount#2838, try_cast(improvement_surcharge#2818 as double) AS improvement_surcharge#2839, try_cast(total_amount#2819 as double) AS total_amount#2840, try_cast(congestion_surcharge#2820 as double) AS congestion_surcharge#2841, try_cast(airport_fee#2821 as double) AS airport_fee#2842, current_timestamp() AS current_timestamp()#2823, coalesce(year(cast(tpep_pickup_datetime#2804 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2843, coalesce(month(cast(tpep_pickup_datetime#2804 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2844, coalesce(day(cast(tpep_pickup_date...
|
|
|
[77]
|
[77]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
137 ms
|
141 ms
|
sql_command {
input {
common {
plan_id: 78
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-11.parquet`\n "
}
}
}
|
CLOSED
|
57bf1364-3172-4342-98f8-e94aa788a939
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_57bf1364-3172-4342-98f8-e94aa788a939
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3375 as bigint) AS VendorID#3397L, try_cast(tpep_pickup_datetime#3376 as timestamp) AS tpep_pickup_datetime#3398, try_cast(tpep_dropoff_datetime#3377 as timestamp) AS tpep_dropoff_datetime#3399, try_cast(passenger_count#3378L as bigint) AS passenger_count#3400L, try_cast(trip_distance#3379 as double) AS trip_distance#3401, try_cast(RatecodeID#3380L as bigint) AS RatecodeID#3402L, try_cast(store_and_fwd_flag#3381 as string) AS store_and_fwd_flag#3403, try_cast(PULocationID#3382 as bigint) AS PULocationID#3404L, try_cast(DOLocationID#3383 as bigint) AS DOLocationID#3405L, try_cast(payment_type#3384L as bigint) AS payment_type#3406L, try_cast(fare_amount#3385 as double) AS fare_amount#3407, try_cast(extra#3386 as double) AS extra#3408, try_cast(mta_tax#3387 as double) AS mta_tax#3409, try_cast(tip_amount#3388 as double) AS tip_amount#3410, try_cast(tolls_amount#3389 as double) AS tolls_amount#3411, try_cast(improvement_surcharge#3390 as double) AS improvement_surcharge#3412, try_cast(total_amount#3391 as double) AS total_amount#3413, try_cast(congestion_surcharge#3392 as double) AS congestion_surcharge#3414, try_cast(airport_fee#3393 as double) AS airport_fee#3415, current_timestamp() AS current_timestamp()#3396, coalesce(year(cast(tpep_pickup_datetime#3376 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3416, coalesce(month(cast(tpep_pickup_datetime#3376 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3417, coalesce(day(cast(tpep_pickup_date...
|
|
|
[193]
|
[199]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
136 ms
|
141 ms
|
write_operation {
input {
common {
plan_id: 62
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
11505f12-b6ea-4784-8780-c85845c387a8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_11505f12-b6ea-4784-8780-c85845c387a8
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[230]
|
[236]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
134 ms
|
137 ms
|
write_operation {
input {
common {
plan_id: 134
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2b2caa6f-19ec-4b1d-a859-3e6126e4c68b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2b2caa6f-19ec-4b1d-a859-3e6126e4c68b
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[202]
|
[208]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
124 ms
|
133 ms
|
write_operation {
input {
common {
plan_id: 80
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d3e4ca73-208a-48c3-bbdf-5076a7d99caa
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d3e4ca73-208a-48c3-bbdf-5076a7d99caa
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[207]
|
[213]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
127 ms
|
131 ms
|
write_operation {
input {
common {
plan_id: 90
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
14c8aa2c-e9b7-4654-8609-907121af310f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_14c8aa2c-e9b7-4654-8609-907121af310f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[68]
|
[68]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
125 ms
|
130 ms
|
sql_command {
input {
common {
plan_id: 69
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-09.parquet`\n "
}
}
}
|
CLOSED
|
63857746-50c6-4e6f-b0a0-3726dd1a630a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_63857746-50c6-4e6f-b0a0-3726dd1a630a
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2979L as bigint) AS VendorID#3000L, try_cast(tpep_pickup_datetime#2980 as timestamp) AS tpep_pickup_datetime#3001, try_cast(tpep_dropoff_datetime#2981 as timestamp) AS tpep_dropoff_datetime#3002, try_cast(passenger_count#2982 as bigint) AS passenger_count#3003L, try_cast(trip_distance#2983 as double) AS trip_distance#3004, try_cast(RatecodeID#2984 as bigint) AS RatecodeID#3005L, try_cast(store_and_fwd_flag#2985 as string) AS store_and_fwd_flag#3006, try_cast(PULocationID#2986L as bigint) AS PULocationID#3007L, try_cast(DOLocationID#2987L as bigint) AS DOLocationID#3008L, try_cast(payment_type#2988L as bigint) AS payment_type#3009L, try_cast(fare_amount#2989 as double) AS fare_amount#3010, try_cast(extra#2990 as double) AS extra#3011, try_cast(mta_tax#2991 as double) AS mta_tax#3012, try_cast(tip_amount#2992 as double) AS tip_amount#3013, try_cast(tolls_amount#2993 as double) AS tolls_amount#3014, try_cast(improvement_surcharge#2994 as double) AS improvement_surcharge#3015, try_cast(total_amount#2995 as double) AS total_amount#3016, try_cast(congestion_surcharge#2996 as double) AS congestion_surcharge#3017, try_cast(airport_fee#2997 as double) AS airport_fee#3018, current_timestamp() AS current_timestamp()#2999, coalesce(year(cast(tpep_pickup_datetime#2980 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3019, coalesce(month(cast(tpep_pickup_datetime#2980 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3020, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[216]
|
[222]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
119 ms
|
128 ms
|
write_operation {
input {
common {
plan_id: 108
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
821d0b0d-41ea-48a1-b8fa-a3c7cf284237
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_821d0b0d-41ea-48a1-b8fa-a3c7cf284237
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[67]
|
[67]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
119 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 68
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-11.parquet`\n "
}
}
}
|
CLOSED
|
3eadc714-e301-4dca-9d15-ca05eb9c2b8a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_3eadc714-e301-4dca-9d15-ca05eb9c2b8a
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2935L as bigint) AS VendorID#2956L, try_cast(tpep_pickup_datetime#2936 as timestamp) AS tpep_pickup_datetime#2957, try_cast(tpep_dropoff_datetime#2937 as timestamp) AS tpep_dropoff_datetime#2958, try_cast(passenger_count#2938 as bigint) AS passenger_count#2959L, try_cast(trip_distance#2939 as double) AS trip_distance#2960, try_cast(RatecodeID#2940 as bigint) AS RatecodeID#2961L, try_cast(store_and_fwd_flag#2941 as string) AS store_and_fwd_flag#2962, try_cast(PULocationID#2942L as bigint) AS PULocationID#2963L, try_cast(DOLocationID#2943L as bigint) AS DOLocationID#2964L, try_cast(payment_type#2944L as bigint) AS payment_type#2965L, try_cast(fare_amount#2945 as double) AS fare_amount#2966, try_cast(extra#2946 as double) AS extra#2967, try_cast(mta_tax#2947 as double) AS mta_tax#2968, try_cast(tip_amount#2948 as double) AS tip_amount#2969, try_cast(tolls_amount#2949 as double) AS tolls_amount#2970, try_cast(improvement_surcharge#2950 as double) AS improvement_surcharge#2971, try_cast(total_amount#2951 as double) AS total_amount#2972, try_cast(congestion_surcharge#2952 as double) AS congestion_surcharge#2973, try_cast(airport_fee#2953 as double) AS airport_fee#2974, current_timestamp() AS current_timestamp()#2955, coalesce(year(cast(tpep_pickup_datetime#2936 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2975, coalesce(month(cast(tpep_pickup_datetime#2936 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2976, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[61]
|
[61]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
120 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 62
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-11.parquet`\n "
}
}
}
|
CLOSED
|
5d5b0d8c-a3a4-4b14-9ec4-126ac9ef9b6c
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_5d5b0d8c-a3a4-4b14-9ec4-126ac9ef9b6c
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2671L as bigint) AS VendorID#2692L, try_cast(tpep_pickup_datetime#2672 as timestamp) AS tpep_pickup_datetime#2693, try_cast(tpep_dropoff_datetime#2673 as timestamp) AS tpep_dropoff_datetime#2694, try_cast(passenger_count#2674 as bigint) AS passenger_count#2695L, try_cast(trip_distance#2675 as double) AS trip_distance#2696, try_cast(RatecodeID#2676 as bigint) AS RatecodeID#2697L, try_cast(store_and_fwd_flag#2677 as string) AS store_and_fwd_flag#2698, try_cast(PULocationID#2678L as bigint) AS PULocationID#2699L, try_cast(DOLocationID#2679L as bigint) AS DOLocationID#2700L, try_cast(payment_type#2680L as bigint) AS payment_type#2701L, try_cast(fare_amount#2681 as double) AS fare_amount#2702, try_cast(extra#2682 as double) AS extra#2703, try_cast(mta_tax#2683 as double) AS mta_tax#2704, try_cast(tip_amount#2684 as double) AS tip_amount#2705, try_cast(tolls_amount#2685 as double) AS tolls_amount#2706, try_cast(improvement_surcharge#2686 as double) AS improvement_surcharge#2707, try_cast(total_amount#2687 as double) AS total_amount#2708, try_cast(congestion_surcharge#2688 as double) AS congestion_surcharge#2709, try_cast(airport_fee#2689 as double) AS airport_fee#2710, current_timestamp() AS current_timestamp()#2691, coalesce(year(cast(tpep_pickup_datetime#2672 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2711, coalesce(month(cast(tpep_pickup_datetime#2672 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2712, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[63]
|
[63]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
117 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 64
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-10.parquet`\n "
}
}
}
|
CLOSED
|
da61d58d-5f1c-4128-bd01-85135b94f575
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_da61d58d-5f1c-4128-bd01-85135b94f575
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2759L as bigint) AS VendorID#2780L, try_cast(tpep_pickup_datetime#2760 as timestamp) AS tpep_pickup_datetime#2781, try_cast(tpep_dropoff_datetime#2761 as timestamp) AS tpep_dropoff_datetime#2782, try_cast(passenger_count#2762 as bigint) AS passenger_count#2783L, try_cast(trip_distance#2763 as double) AS trip_distance#2784, try_cast(RatecodeID#2764 as bigint) AS RatecodeID#2785L, try_cast(store_and_fwd_flag#2765 as string) AS store_and_fwd_flag#2786, try_cast(PULocationID#2766L as bigint) AS PULocationID#2787L, try_cast(DOLocationID#2767L as bigint) AS DOLocationID#2788L, try_cast(payment_type#2768L as bigint) AS payment_type#2789L, try_cast(fare_amount#2769 as double) AS fare_amount#2790, try_cast(extra#2770 as double) AS extra#2791, try_cast(mta_tax#2771 as double) AS mta_tax#2792, try_cast(tip_amount#2772 as double) AS tip_amount#2793, try_cast(tolls_amount#2773 as double) AS tolls_amount#2794, try_cast(improvement_surcharge#2774 as double) AS improvement_surcharge#2795, try_cast(total_amount#2775 as double) AS total_amount#2796, try_cast(congestion_surcharge#2776 as double) AS congestion_surcharge#2797, try_cast(airport_fee#2777 as double) AS airport_fee#2798, current_timestamp() AS current_timestamp()#2779, coalesce(year(cast(tpep_pickup_datetime#2760 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2799, coalesce(month(cast(tpep_pickup_datetime#2760 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2800, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[78]
|
[78]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
122 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 79
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-07.parquet`\n "
}
}
}
|
CLOSED
|
dae7f573-38d6-40b9-a0b7-62773b7ed2d8
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_dae7f573-38d6-40b9-a0b7-62773b7ed2d8
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3420L as bigint) AS VendorID#3441L, try_cast(tpep_pickup_datetime#3421 as timestamp) AS tpep_pickup_datetime#3442, try_cast(tpep_dropoff_datetime#3422 as timestamp) AS tpep_dropoff_datetime#3443, try_cast(passenger_count#3423 as bigint) AS passenger_count#3444L, try_cast(trip_distance#3424 as double) AS trip_distance#3445, try_cast(RatecodeID#3425 as bigint) AS RatecodeID#3446L, try_cast(store_and_fwd_flag#3426 as string) AS store_and_fwd_flag#3447, try_cast(PULocationID#3427L as bigint) AS PULocationID#3448L, try_cast(DOLocationID#3428L as bigint) AS DOLocationID#3449L, try_cast(payment_type#3429L as bigint) AS payment_type#3450L, try_cast(fare_amount#3430 as double) AS fare_amount#3451, try_cast(extra#3431 as double) AS extra#3452, try_cast(mta_tax#3432 as double) AS mta_tax#3453, try_cast(tip_amount#3433 as double) AS tip_amount#3454, try_cast(tolls_amount#3434 as double) AS tolls_amount#3455, try_cast(improvement_surcharge#3435 as double) AS improvement_surcharge#3456, try_cast(total_amount#3436 as double) AS total_amount#3457, try_cast(congestion_surcharge#3437 as double) AS congestion_surcharge#3458, try_cast(airport_fee#3438 as double) AS airport_fee#3459, current_timestamp() AS current_timestamp()#3440, coalesce(year(cast(tpep_pickup_datetime#3421 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3460, coalesce(month(cast(tpep_pickup_datetime#3421 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3461, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[236]
|
[242]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
121 ms
|
127 ms
|
write_operation {
input {
common {
plan_id: 146
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a096a92e-2cf9-4190-bd0c-07e3dea8fab7
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a096a92e-2cf9-4190-bd0c-07e3dea8fab7
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[93]
|
[93]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
105 ms
|
125 ms
|
sql_command {
input {
common {
plan_id: 94
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-04.parquet`\n "
}
}
}
|
CLOSED
|
4f6df773-6d3f-428e-9bfc-d72f445d2d47
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4f6df773-6d3f-428e-9bfc-d72f445d2d47
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#4081L as bigint) AS VendorID#4102L, try_cast(tpep_pickup_datetime#4082 as timestamp) AS tpep_pickup_datetime#4103, try_cast(tpep_dropoff_datetime#4083 as timestamp) AS tpep_dropoff_datetime#4104, try_cast(passenger_count#4084 as bigint) AS passenger_count#4105L, try_cast(trip_distance#4085 as double) AS trip_distance#4106, try_cast(RatecodeID#4086 as bigint) AS RatecodeID#4107L, try_cast(store_and_fwd_flag#4087 as string) AS store_and_fwd_flag#4108, try_cast(PULocationID#4088L as bigint) AS PULocationID#4109L, try_cast(DOLocationID#4089L as bigint) AS DOLocationID#4110L, try_cast(payment_type#4090L as bigint) AS payment_type#4111L, try_cast(fare_amount#4091 as double) AS fare_amount#4112, try_cast(extra#4092 as double) AS extra#4113, try_cast(mta_tax#4093 as double) AS mta_tax#4114, try_cast(tip_amount#4094 as double) AS tip_amount#4115, try_cast(tolls_amount#4095 as double) AS tolls_amount#4116, try_cast(improvement_surcharge#4096 as double) AS improvement_surcharge#4117, try_cast(total_amount#4097 as double) AS total_amount#4118, try_cast(congestion_surcharge#4098 as double) AS congestion_surcharge#4119, try_cast(airport_fee#4099 as double) AS airport_fee#4120, current_timestamp() AS current_timestamp()#4101, coalesce(year(cast(tpep_pickup_datetime#4082 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4121, coalesce(month(cast(tpep_pickup_datetime#4082 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4122, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[87]
|
[87]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
120 ms
|
125 ms
|
sql_command {
input {
common {
plan_id: 88
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-12.parquet`\n "
}
}
}
|
CLOSED
|
b2bd0011-a204-4aa2-bb89-081f8b518c8b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_b2bd0011-a204-4aa2-bb89-081f8b518c8b
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3816L as bigint) AS VendorID#3837L, try_cast(tpep_pickup_datetime#3817 as timestamp) AS tpep_pickup_datetime#3838, try_cast(tpep_dropoff_datetime#3818 as timestamp) AS tpep_dropoff_datetime#3839, try_cast(passenger_count#3819 as bigint) AS passenger_count#3840L, try_cast(trip_distance#3820 as double) AS trip_distance#3841, try_cast(RatecodeID#3821 as bigint) AS RatecodeID#3842L, try_cast(store_and_fwd_flag#3822 as string) AS store_and_fwd_flag#3843, try_cast(PULocationID#3823L as bigint) AS PULocationID#3844L, try_cast(DOLocationID#3824L as bigint) AS DOLocationID#3845L, try_cast(payment_type#3825L as bigint) AS payment_type#3846L, try_cast(fare_amount#3826 as double) AS fare_amount#3847, try_cast(extra#3827 as double) AS extra#3848, try_cast(mta_tax#3828 as double) AS mta_tax#3849, try_cast(tip_amount#3829 as double) AS tip_amount#3850, try_cast(tolls_amount#3830 as double) AS tolls_amount#3851, try_cast(improvement_surcharge#3831 as double) AS improvement_surcharge#3852, try_cast(total_amount#3832 as double) AS total_amount#3853, try_cast(congestion_surcharge#3833 as double) AS congestion_surcharge#3854, try_cast(airport_fee#3834 as double) AS airport_fee#3855, current_timestamp() AS current_timestamp()#3836, coalesce(year(cast(tpep_pickup_datetime#3817 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3856, coalesce(month(cast(tpep_pickup_datetime#3817 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3857, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[86]
|
[86]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
117 ms
|
125 ms
|
sql_command {
input {
common {
plan_id: 87
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-08.parquet`\n "
}
}
}
|
CLOSED
|
b67a5f83-a293-4cb0-a08f-01c601ab8aab
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_b67a5f83-a293-4cb0-a08f-01c601ab8aab
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3772L as bigint) AS VendorID#3793L, try_cast(tpep_pickup_datetime#3773 as timestamp) AS tpep_pickup_datetime#3794, try_cast(tpep_dropoff_datetime#3774 as timestamp) AS tpep_dropoff_datetime#3795, try_cast(passenger_count#3775 as bigint) AS passenger_count#3796L, try_cast(trip_distance#3776 as double) AS trip_distance#3797, try_cast(RatecodeID#3777 as bigint) AS RatecodeID#3798L, try_cast(store_and_fwd_flag#3778 as string) AS store_and_fwd_flag#3799, try_cast(PULocationID#3779L as bigint) AS PULocationID#3800L, try_cast(DOLocationID#3780L as bigint) AS DOLocationID#3801L, try_cast(payment_type#3781L as bigint) AS payment_type#3802L, try_cast(fare_amount#3782 as double) AS fare_amount#3803, try_cast(extra#3783 as double) AS extra#3804, try_cast(mta_tax#3784 as double) AS mta_tax#3805, try_cast(tip_amount#3785 as double) AS tip_amount#3806, try_cast(tolls_amount#3786 as double) AS tolls_amount#3807, try_cast(improvement_surcharge#3787 as double) AS improvement_surcharge#3808, try_cast(total_amount#3788 as double) AS total_amount#3809, try_cast(congestion_surcharge#3789 as double) AS congestion_surcharge#3810, try_cast(airport_fee#3790 as double) AS airport_fee#3811, current_timestamp() AS current_timestamp()#3792, coalesce(year(cast(tpep_pickup_datetime#3773 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3812, coalesce(month(cast(tpep_pickup_datetime#3773 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3813, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[84]
|
[84]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
112 ms
|
124 ms
|
sql_command {
input {
common {
plan_id: 85
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-03.parquet`\n "
}
}
}
|
CLOSED
|
abff517f-610b-4643-a428-3d861cc6fa42
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_abff517f-610b-4643-a428-3d861cc6fa42
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3684L as bigint) AS VendorID#3705L, try_cast(tpep_pickup_datetime#3685 as timestamp) AS tpep_pickup_datetime#3706, try_cast(tpep_dropoff_datetime#3686 as timestamp) AS tpep_dropoff_datetime#3707, try_cast(passenger_count#3687 as bigint) AS passenger_count#3708L, try_cast(trip_distance#3688 as double) AS trip_distance#3709, try_cast(RatecodeID#3689 as bigint) AS RatecodeID#3710L, try_cast(store_and_fwd_flag#3690 as string) AS store_and_fwd_flag#3711, try_cast(PULocationID#3691L as bigint) AS PULocationID#3712L, try_cast(DOLocationID#3692L as bigint) AS DOLocationID#3713L, try_cast(payment_type#3693L as bigint) AS payment_type#3714L, try_cast(fare_amount#3694 as double) AS fare_amount#3715, try_cast(extra#3695 as double) AS extra#3716, try_cast(mta_tax#3696 as double) AS mta_tax#3717, try_cast(tip_amount#3697 as double) AS tip_amount#3718, try_cast(tolls_amount#3698 as double) AS tolls_amount#3719, try_cast(improvement_surcharge#3699 as double) AS improvement_surcharge#3720, try_cast(total_amount#3700 as double) AS total_amount#3721, try_cast(congestion_surcharge#3701 as double) AS congestion_surcharge#3722, try_cast(airport_fee#3702 as double) AS airport_fee#3723, current_timestamp() AS current_timestamp()#3704, coalesce(year(cast(tpep_pickup_datetime#3685 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3724, coalesce(month(cast(tpep_pickup_datetime#3685 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3725, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[188]
|
[194]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
116 ms
|
123 ms
|
write_operation {
input {
common {
plan_id: 52
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[62]
|
[62]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
113 ms
|
122 ms
|
sql_command {
input {
common {
plan_id: 63
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-09.parquet`\n "
}
}
}
|
CLOSED
|
2df7501f-cf45-447d-8e5b-5abb0eb1e7dc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_2df7501f-cf45-447d-8e5b-5abb0eb1e7dc
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2715 as bigint) AS VendorID#2736L, try_cast(tpep_pickup_datetime#2716 as timestamp) AS tpep_pickup_datetime#2737, try_cast(tpep_dropoff_datetime#2717 as timestamp) AS tpep_dropoff_datetime#2738, try_cast(passenger_count#2718L as bigint) AS passenger_count#2739L, try_cast(trip_distance#2719 as double) AS trip_distance#2740, try_cast(RatecodeID#2720L as bigint) AS RatecodeID#2741L, try_cast(store_and_fwd_flag#2721 as string) AS store_and_fwd_flag#2742, try_cast(PULocationID#2722 as bigint) AS PULocationID#2743L, try_cast(DOLocationID#2723 as bigint) AS DOLocationID#2744L, try_cast(payment_type#2724L as bigint) AS payment_type#2745L, try_cast(fare_amount#2725 as double) AS fare_amount#2746, try_cast(extra#2726 as double) AS extra#2747, try_cast(mta_tax#2727 as double) AS mta_tax#2748, try_cast(tip_amount#2728 as double) AS tip_amount#2749, try_cast(tolls_amount#2729 as double) AS tolls_amount#2750, try_cast(improvement_surcharge#2730 as double) AS improvement_surcharge#2751, try_cast(total_amount#2731 as double) AS total_amount#2752, try_cast(congestion_surcharge#2732 as double) AS congestion_surcharge#2753, try_cast(airport_fee#2733 as double) AS airport_fee#2754, current_timestamp() AS current_timestamp()#2735, coalesce(year(cast(tpep_pickup_datetime#2716 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2755, coalesce(month(cast(tpep_pickup_datetime#2716 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2756, coalesce(day(cast(tpep_pickup_date...
|