Spark Connect server - SQL / DataFrame

Completed Queries: 15
Failed Queries: 221

Completed Queries (15)

Page:

ID ▾

Description

Submitted

Duration

Job IDs

Sub Execution IDs

225

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 114
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "21211c3d-a004-4101-a1bb-488c89fddc2f"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:39

27 s

[226]

+details

Description

Submitted

Duration

Job IDs

226

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 114
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "21211c3d-a004-4101-a1bb-488c89fddc2f"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:39

27 s

[220]

175

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 16
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "035259f2-8534-4df9-9603-193cb3db6fe8"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:08

25 s

[176]

+details

Description

Submitted

Duration

Job IDs

176

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 16
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "035259f2-8534-4df9-9603-193cb3db6fe8"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:08

25 s

[170]

170

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 8
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4a0e6fa4-11f8-45c5-9138-12142a2c4d93"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:12:39

28 s

[171]

+details

Description

Submitted

Duration

Job IDs

171

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 8
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4a0e6fa4-11f8-45c5-9138-12142a2c4d93"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:12:39

27 s

[165]

164

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 0
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "72c531e6-b91e-430c-a90f-107332e492cb"

2026/09/03 16:12:07

32 s

[165][166]

+details

Description

Submitted

Duration

Job IDs

165

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 0
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "72c531e6-b91e-430c-a90f-107332e492cb"

2026/09/03 16:12:07

31 s

[159]

166

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 0
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "72c531e6-b91e-430c-a90f-107332e492cb"

2026/09/03 16:12:38

0.2 s

[160]

161

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 189
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0d73ec80-efc3-457f-b7a0-ecf03792de7e"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:27

13 s

[162][163]

+details

Description

Submitted

Duration

Job IDs

162

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 189
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0d73ec80-efc3-457f-b7a0-ecf03792de7e"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:27

12 s

[156]

163

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 189
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0d73ec80-efc3-457f-b7a0-ecf03792de7e"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:39

0.2 s

[157]

111

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 116 } aggregate { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 116
    }
    aggregate {
    ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 116
    }
    aggregate {
      input {
        common {
          plan_id: 115
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      group_type: GROUP_TYPE_GROUPBY
      aggregate_expressions {
        unresolved_function {
          function_name: "count"
          arguments {
            literal {
              integer: 1
            }
          }
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "1534209b-e69e-4668-879a-4f7b18f0c8ed"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:04:22

1 s

[104][105]

110

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 114 } aggregate { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 114
    }
    aggregate {
    ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 114
    }
    aggregate {
      input {
        common {
          plan_id: 113
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      group_type: GROUP_TYPE_GROUPBY
      aggregate_expressions {
        unresolved_function {
          function_name: "count"
          arguments {
            literal {
              integer: 1
            }
          }
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "7e420069-8d5c-4a04-8ffd-b5541331883b"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:03:59

2 s

[102][103]

109

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 112 } limit { in... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      in...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      input {
        common {
          plan_id: 111
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      limit: 1000
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5c4d2c10-fdcb-48fb-b90d-06cbd97c3009"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:03:42

2 s

[101]

108

Delta: Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 112 } limit { in...: Filtering files for query: Compute snapshot for version: 2 +details

Delta: Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      in...: Filtering files for query: Compute snapshot for version: 2

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      input {
        common {
          plan_id: 111
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      limit: 1000
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5c4d2c10-fdcb-48fb-b90d-06cbd97c3009"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:03:36

5 s

[99][100]

107

Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 112 } limit { in...: Filtering files for query +details

Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      in...: Filtering files for query

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      input {
        common {
          plan_id: 111
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      limit: 1000
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5c4d2c10-fdcb-48fb-b90d-06cbd97c3009"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:03:26

10 s

[98]

106

Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 112 } limit { in...: Filtering files for query +details

Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      in...: Filtering files for query

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      input {
        common {
          plan_id: 111
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      limit: 1000
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5c4d2c10-fdcb-48fb-b90d-06cbd97c3009"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:03:25

0.2 s

105

Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { root { common { plan_id: 112 } limit { in...: Filtering files for query +details

Delta: Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      in...: Filtering files for query

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  root {
    common {
      plan_id: 112
    }
    limit {
      input {
        common {
          plan_id: 111
        }
        read {
          data_source {
            format: "delta"
            schema: ""
            paths: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips"
          }
        }
      }
      limit: 1000
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5c4d2c10-fdcb-48fb-b90d-06cbd97c3009"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:03:19

6 s

[97]

101

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { sql_command { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
      ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
          plan_id: 105
        }
        sql {
          query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n    VendorID long,\n    tpep_pickup_datetime timestamp,\n    tpep_dropoff_datetime timestamp,\n    passenger_count long,\n    trip_distance double,\n    RatecodeID long,\n    store_and_fwd_flag string,\n    PULocationID long,\n    DOLocationID long,\n    payment_type long,\n    fare_amount double,\n    extra double,\n    mta_tax double,\n    tip_amount double,\n    tolls_amount double,\n    improvement_surcharge double,\n    total_amount double,\n    congestion_surcharge double,\n    airport_fee double,\n    load_timestamp timestamp,\n    year int,\n    month int,\n    day int\n)\nusing delta\npartitioned by (year,month)\nlocation \'/home/jovyan/spark-warehouse/yellow_taxi.db/trips\'\n"
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4e3e69c5-e8a8-4c00-a30a-7571f47e1126"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 15:59:02

81 ms

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { sql_command { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
      ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
          plan_id: 99
        }
        sql {
          query: "create schema if not exists spark_catalog.yellow_taxi"
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3322b461-013d-4354-aa6f-30d2254e748c"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 15:56:09

11 ms

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { create_dataframe_view { input { commo... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    create_dataframe_view {
      input {
        commo...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    create_dataframe_view {
      input {
        common {
          plan_id: 0
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi"
          }
        }
      }
      name: "trips"
      replace: true
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3f499705-fda8-4e45-848f-dae0448e100f"

2026/09/03 15:55:27

0.5 s

Page:

Failed Queries (221)

Page:

Description

Submitted

Duration ▾

Succeeded Job IDs

Failed Job IDs

Error Message

Sub Execution IDs

112

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 117
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "99accdac-7c3e-412d-adf9-fcb58b5c916c"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:09:50

10 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[113]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

113

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 117
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "99accdac-7c3e-412d-adf9-fcb58b5c916c"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:09:50

10 s

[107]

155

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 183
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "19e2d57c-1fb5-4d37-a377-c4460bc08d30"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

7 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[156]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

156

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 183
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "19e2d57c-1fb5-4d37-a377-c4460bc08d30"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

7 s

[150]

157

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 185
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "a1541722-9f42-44d1-b623-e01e03ca7219"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:17

5 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[158]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

158

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 185
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "a1541722-9f42-44d1-b623-e01e03ca7219"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:17

5 s

[152]

114

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 119
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4834ae02-5360-4c58-8f85-4f9f8f34dae7"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:00

4 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[115]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

115

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 119
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4834ae02-5360-4c58-8f85-4f9f8f34dae7"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:00

3 s

[109]

159

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 187
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "ce16240e-152e-4179-aeb2-45a9b7a0bbbf"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:23

4 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[160]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

160

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 187
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "ce16240e-152e-4179-aeb2-45a9b7a0bbbf"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:23

4 s

[154]

116

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 121
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "f4ab84f4-63f7-4ffd-9e57-1bc858c28878"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:04

0.8 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[117]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

117

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 121
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "f4ab84f4-63f7-4ffd-9e57-1bc858c28878"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:04

0.5 s

[111]

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { sql_command { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
      ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
          plan_id: 102
        }
        sql {
          query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n    VendorID long,\n    tpep_pickup_datetime timestamp,\n    tpep_dropoff_datetime timestamp,\n    passenger_count long,\n    trip_distance double,\n    RatecodeID long,\n    store_and_fwd_flag string,\n    PULocationID long,\n    DOLocationID long,\n    payment_type long,\n    fare_amount double,\n    extra double,\n    mta_tax double,\n    tip_amount double,\n    tolls_amount double,\n    improvement_surcharge double,\n    total_amount double,\n    congestion_surcharge double,\n    airport_fee double,\n    load_timestamp timestamp,\n    year int,\n    month int,\n    day int\n)\nusing delta\npartitioned by (year,month)\n"
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "9fcede98-796d-4cc6-b745-53573fede9fd"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 15:56:11

0.7 s

DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION +details

[DELTA_CREATE_TABLE_WITH_NON_EMPTY_LOCATION] Cannot create table ('`yellow_taxi`.`trips`'). The associated location ('file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips') is not empty and also not a Delta table.

147

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 167
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0e0ae871-5c31-43fe-bfcd-3145d557e602"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:09

0.4 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[148]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

148

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 167
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0e0ae871-5c31-43fe-bfcd-3145d557e602"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:09

0.2 s

125

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 133
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3353fded-646f-4e2b-94ee-ddf45ec66e18"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[126]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

126

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 133
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3353fded-646f-4e2b-94ee-ddf45ec66e18"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

47 ms

[119]

138

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 155
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4909aecf-283f-47b5-8b1a-eaca18732b21"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[139]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

139

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 155
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4909aecf-283f-47b5-8b1a-eaca18732b21"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

58 ms

134

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 147
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "14b3700d-9902-44c3-8c7b-7d1ab2be7dac"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[135]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

135

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 147
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "14b3700d-9902-44c3-8c7b-7d1ab2be7dac"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

61 ms

145

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 165
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "2f228dd1-be15-416d-be3d-8dc0abcdbd80"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:09

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[146]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

146

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 165
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "2f228dd1-be15-416d-be3d-8dc0abcdbd80"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:09

0.1 s

[139]

119

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 127
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "24da78ab-53f3-47e8-b928-e057f6c960f7"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:05

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[120]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

120

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 127
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "24da78ab-53f3-47e8-b928-e057f6c960f7"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

63 ms

152

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 179
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4a4d2cea-6df1-4258-81eb-757d681d47f8"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[153]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

153

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 179
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4a4d2cea-6df1-4258-81eb-757d681d47f8"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

78 ms

[148]

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { sql_command { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
      ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
          plan_id: 98
        }
        sql {
          query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n    VendorID long,\n    tpep_pickup_datetime timestamp,\n    tpep_dropoff_datetime timestamp,\n    passenger_count long,\n    trip_distance double,\n    RatecodeID long,\n    store_and_fwd_flag string,\n    PULocationID long,\n    DOLocationID long,\n    payment_type long,\n    fare_amount double,\n    extra double,\n    mta_tax double,\n    tip_amount double,\n    tolls_amount double,\n    improvement_surcharge double,\n    total_amount double,\n    congestion_surcharge double,\n    airport_fee double,\n    load_timestamp timestamp,\n    year int,\n    month int,\n    day int\n)\nusing delta\npartitioned by (year,month)\n"
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "a13d02e2-eec6-48b6-814e-ba08e461b69e"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 15:55:59

0.2 s

SCHEMA_NOT_FOUND +details

[SCHEMA_NOT_FOUND] The schema `spark_catalog`.`yellow_taxi` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a catalog, verify the current_schema() output, or qualify the name with the correct catalog.
To tolerate the error on drop use DROP SCHEMA IF EXISTS. SQLSTATE: 42704

121

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 129
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "6780f38f-c162-42bf-98a5-81e0579a82ee"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[122]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

122

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 129
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "6780f38f-c162-42bf-98a5-81e0579a82ee"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

50 ms

123

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 131
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "bf40aa6a-2502-42af-8fda-de65e57da0fc"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[124]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

124

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 131
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "bf40aa6a-2502-42af-8fda-de65e57da0fc"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

53 ms

150

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 177
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4f641188-f012-4bf6-bce6-417e8736f9c1"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

0.2 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[151]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

151

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 177
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4f641188-f012-4bf6-bce6-417e8736f9c1"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

45 ms

[146]

136

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 151
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "7addeaeb-55db-4d99-a6af-39b0ebebf3da"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

0.1 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[137]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

137

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 151
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "7addeaeb-55db-4d99-a6af-39b0ebebf3da"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

59 ms

[129]

141

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 161
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e9f70b88-ec0d-429a-ae12-eef385da88ec"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

0.1 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[142]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

142

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 161
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e9f70b88-ec0d-429a-ae12-eef385da88ec"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

38 ms

132

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 143
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "c01a3fb1-bdc7-4ec1-a598-e4f540be20fe"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

0.1 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[133]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

133

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 143
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "c01a3fb1-bdc7-4ec1-a598-e4f540be20fe"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

55 ms

[124]

143

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 163
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "c2fa0c35-4ecf-4024-9fd5-3129c9236878"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

0.1 s

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1134)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:253)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:265)
	at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:104)
	at org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:391)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1006)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2496)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:269)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[144]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

144

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 163
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "c2fa0c35-4ecf-4024-9fd5-3129c9236878"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

47 ms

[136]

130

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 141
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0736c6a9-8500-4c2b-a735-a0089e70f6bc"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

95 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[131]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

131

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 141
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0736c6a9-8500-4c2b-a735-a0089e70f6bc"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:07

34 ms

128

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 139
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e9888ae1-7199-4cff-9c64-58021fbdf24e"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

91 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1167)
	at scala.concurrent.impl.Promise$DefaultPromise.tryAwait0(Promise.scala:251)
	at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:271)
	at org.apache.spark.util.SparkThreadUtils$.awaitResultNoSparkExceptionConversion(SparkThreadUtils.scala:61)
	at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:45)
	at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:359)
	at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101)
	at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85)
	at org.apache.spark.storage.BlockManagerMaster.updateBlockInfo(BlockManagerMaster.scala:106)
	at org.apache.spark.storage.BlockManager.tryToReportBlockStatus(BlockManager.scala:905)
	at org.apache.spark.storage.BlockManager.reportBlockStatus(BlockManager.scala:881)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.$anonfun$save$1(BlockManager.scala:444)
	at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1585)
	at org.apache.spark.storage.BlockManager$BlockStoreUpdater.save(BlockManager.scala:408)
	at org.apache.spark.storage.BlockManager.putBytes(BlockManager.scala:1528)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1(TorrentBroadcast.scala:172)
	at org.apache.spark.broadcast.TorrentBroadcast.$anonfun$writeBlocks$1$adapted(TorrentBroadcast.scala:166)
	at scala.collection.ArrayOps$.foreach$extension(ArrayOps.scala:1324)
	at org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:166)
	at org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:100)
	at org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:38)
	at org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:78)
	at org.apache.spark.SparkContext.broadcastInternal(SparkContext.scala:1739)
	at org.apache.spark.SparkContext.broadcast(SparkContext.scala:1721)
	at org.apache.spark.util.SerializableConfiguration$.broadcast(SerializableConfiguration.scala:47)
	at org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.buildReaderWithPartitionValues(ParquetFileFormat.scala:177)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD$lzycompute(DataSourceScanExec.scala:728)
	at org.apache.spark.sql.execution.FileSourceScanExec.inputRDD(DataSourceScanExec.scala:717)
	at org.apache.spark.sql.execution.FileSourceScanExec.doExecuteColumnar(DataSourceScanExec.scala:767)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.InputAdapter.doExecuteColumnar(WholeStageCodegenExec.scala:527)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnarRDD$1(SparkPlan.scala:222)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeColumnar$1(SparkPlan.scala:236)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.executeColumnar(SparkPlan.scala:232)
	at org.apache.spark.sql.execution.ColumnarToRowExec.inputRDDs(Columnar.scala:205)
	at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:763)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:85)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeRDD$1(SparkPlan.scala:188)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:201)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:260)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:257)
	at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:197)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.$anonfun$executeWrite$1(DeltaFileFormatWriter.scala:255)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.writeAndCommit(DeltaFileFormatWriter.scala:302)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.executeWrite(DeltaFileFormatWriter.scala:238)
	at org.apache.spark.sql.delta.files.DeltaFileFormatWriter$.write(DeltaFileFormatWriter.scala:218)
	at org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$2(TransactionalWrite.scala:490)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:445)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:402)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:264)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:260)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:253)
	at org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:250)
	at org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:170)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeFiles(WriteIntoDelta.scala:388)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.writeAndReturnCommitData(WriteIntoDelta.scala:348)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:110)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:249)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

[129]

+details

Description

Submitted

Duration

Succeeded Job IDs

Failed Job IDs

Error Message

129

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 139
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e9888ae1-7199-4cff-9c64-58021fbdf24e"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

30 ms

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { sql_command { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
      ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
          plan_id: 15
        }
        sql {
          query: "insert into yellow_taxi.trips\n        select \n            try_cast(VendorID as long),\n            try_cast(tpep_pickup_datetime as timestamp),\n            try_cast(tpep_dropoff_datetime as timestamp),\n            try_cast(passenger_count as long),\n            try_cast(trip_distance as double),\n            try_cast(RatecodeID as long),\n            try_cast(store_and_fwd_flag as string),\n            try_cast(PULocationID as long),\n            try_cast(DOLocationID as long),\n            try_cast(payment_type as long),\n            try_cast(fare_amount as double),\n            try_cast(extra as double),\n            try_cast(mta_tax as double),\n            try_cast(tip_amount as double),\n            try_cast(tolls_amount as double),\n            try_cast(improvement_surcharge as double),\n            try_cast(total_amount as double),\n            try_cast(congestion_surcharge as double),\n            try_cast(airport_fee as double),\n            current_timestamp,\n            coalesce(year(tpep_pickup_datetime),2026),\n  [truncated(size=1,230)]"
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5626a84d-cd61-4f83-826f-9ccbf49fbe03"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 15:55:46

29 ms

TABLE_OR_VIEW_NOT_FOUND +details

[TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#594 as bigint) AS VendorID#615L, try_cast(tpep_pickup_datetime#595 as timestamp) AS tpep_pickup_datetime#616, try_cast(tpep_dropoff_datetime#596 as timestamp) AS tpep_dropoff_datetime#617, try_cast(passenger_count#597L as bigint) AS passenger_count#618L, try_cast(trip_distance#598 as double) AS trip_distance#619, try_cast(RatecodeID#599L as bigint) AS RatecodeID#620L, try_cast(store_and_fwd_flag#600 as string) AS store_and_fwd_flag#621, try_cast(PULocationID#601 as bigint) AS PULocationID#622L, try_cast(DOLocationID#602 as bigint) AS DOLocationID#623L, try_cast(payment_type#603L as bigint) AS payment_type#624L, try_cast(fare_amount#604 as double) AS fare_amount#625, try_cast(extra#605 as double) AS extra#626, try_cast(mta_tax#606 as double) AS mta_tax#627, try_cast(tip_amount#607 as double) AS tip_amount#628, try_cast(tolls_amount#608 as double) AS tolls_amount#629, try_cast(improvement_surcharge#609 as double) AS improvement_surcharge#630, try_cast(total_amount#610 as double) AS total_amount#631, try_cast(congestion_surcharge#611 as double) AS congestion_surcharge#632, try_cast(airport_fee#612 as double) AS airport_fee#633, current_timestamp() AS current_timestamp()#614, coalesce(year(cast(tpep_pickup_datetime#595 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#634, coalesce(month(cast(tpep_pickup_datetime#595 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#635, coalesce(day(cast(tpep_pickup_datetime#595 as date)), 3) AS coalesce(day(tpep_pickup_datetime), 3)#636]
   +- Relation [VendorID#594,tpep_pickup_datetime#595,tpep_dropoff_datetime#596,passenger_count#597L,trip_distance#598,RatecodeID#599L,store_and_fwd_flag#600,PULocationID#601,DOLocationID#602,payment_type#603L,fare_amount#604,extra#605,mta_tax#606,tip_amount#607,tolls_amount#608,improvement_surcharge#609,total_amount#610,congestion_surcharge#611,Airport_fee#612] parquet

118

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 125
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "30c3a325-39ef-4d92-aeb2-0c3cb47450ee"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:05

28 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.ReentrantLock$Sync.lockInterruptibly(ReentrantLock.java:159)
	at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:372)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly(SnapshotManagement.scala:86)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly$(SnapshotManagement.scala:85)
	at org.apache.spark.sql.delta.DeltaLog.withSnapshotLockInterruptibly(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$2(SnapshotManagement.scala:1092)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:171)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:169)
	at org.apache.spark.sql.delta.DeltaLog.recordFrameProfile(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:1091)
	at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:1059)
	at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at scala.Option.getOrElse(Option.scala:201)
	at org.apache.spark.sql.delta.OptimisticTransaction.<init>(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.DeltaLog.$anonfun$startTransaction$1(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.NoOpTransactionExecutionObserver$.startingTransaction(TransactionExecutionObserver.scala:109)
	at org.apache.spark.sql.delta.DeltaLog.startTransaction(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:246)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

198

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 60
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "2c86dd1a-1570-4f98-9e2e-9b81a80d2006"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

28 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

211

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 86
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "d7ae4bfa-6ae1-4130-a89f-a2a3062ce403"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:37

26 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

250

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 162
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "f988164d-2a30-4c6c-812b-4ad65094c526"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:09

26 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'

251

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 164
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "68456b2c-116e-405c-9494-0c45481a30f9"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:09

26 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

173

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 12
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "ccc38014-821f-42c5-b1f5-65d5700fbadf"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:07

23 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

190

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 44
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "7a5df2e0-b57e-470b-94e5-badc2b56aa44"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

23 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

185

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 34
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "58782597-3c4f-4429-82f1-d7be3d74c34e"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:34

21 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

208

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 80
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "d3e4ca73-208a-48c3-bbdf-5076a7d99caa"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:37

20 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

177

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 18
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "ce551e70-2c11-40d6-bee9-c12c33e80063"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:33

18 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

149

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 175
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3bb4164c-8dcf-4a51-bc6d-b9c29174b839"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

17 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.ReentrantLock$Sync.lockInterruptibly(ReentrantLock.java:159)
	at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:372)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly(SnapshotManagement.scala:86)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly$(SnapshotManagement.scala:85)
	at org.apache.spark.sql.delta.DeltaLog.withSnapshotLockInterruptibly(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$2(SnapshotManagement.scala:1092)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:171)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:169)
	at org.apache.spark.sql.delta.DeltaLog.recordFrameProfile(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:1091)
	at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:1059)
	at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at scala.Option.getOrElse(Option.scala:201)
	at org.apache.spark.sql.delta.OptimisticTransaction.<init>(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.DeltaLog.$anonfun$startTransaction$1(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.NoOpTransactionExecutionObserver$.startingTransaction(TransactionExecutionObserver.scala:109)
	at org.apache.spark.sql.delta.DeltaLog.startTransaction(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:246)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

154

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 181
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3f39aeed-ff3c-4900-8789-60d0940d1a76"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:10

17 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.ReentrantLock$Sync.lockInterruptibly(ReentrantLock.java:159)
	at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:372)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly(SnapshotManagement.scala:86)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly$(SnapshotManagement.scala:85)
	at org.apache.spark.sql.delta.DeltaLog.withSnapshotLockInterruptibly(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$2(SnapshotManagement.scala:1092)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:171)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:169)
	at org.apache.spark.sql.delta.DeltaLog.recordFrameProfile(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:1091)
	at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:1059)
	at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at scala.Option.getOrElse(Option.scala:201)
	at org.apache.spark.sql.delta.OptimisticTransaction.<init>(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.DeltaLog.$anonfun$startTransaction$1(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.NoOpTransactionExecutionObserver$.startingTransaction(TransactionExecutionObserver.scala:109)
	at org.apache.spark.sql.delta.DeltaLog.startTransaction(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:246)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

172

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 10
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "95614555-3777-4dfe-826f-dd28d83359e8"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:07

17 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

238

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 138
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "52a96a13-df5e-4a95-af5a-163a6252c863"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:07

17 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

209

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 82
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "b69fd70b-bb76-4198-91e3-0f7223de0693"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:37

16 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

228

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 118
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e95632ff-dda1-4fe8-90ec-9a939137983b"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:06

16 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

174

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 14
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "d2e00546-45c8-4a1f-890f-63c0315ed1cc"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:07

15 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

187

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 38
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "19df5b34-f378-494e-81db-e3371ab2c257"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:34

15 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

188

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 40
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "a9b43a10-8ee3-4cdc-b188-b967035b5a2e"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:34

15 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

242

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 146
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "a096a92e-2cf9-4190-bd0c-07e3dea8fab7"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:08

15 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

255

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 172
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "dd6f855b-4b32-4fb2-8d11-f133ff6422b0"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:09

15 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

181

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 26
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "43c883ae-8661-4fae-9072-fff947596643"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:33

14 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

201

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 66
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "1842e87c-6411-4588-a7cc-b4a675f5ab04"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

14 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

216

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 96
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3c9481a0-0f2d-428a-9746-fcd72062da9f"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

14 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

197

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 58
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "89c328c3-476f-4d77-ae7a-980f234b092b"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

13 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

207

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 78
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "be22460e-a3d6-4c1d-98e9-0f8c38a34805"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:37

13 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

215

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 94
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "00f8c9d7-c11a-411f-b7be-c0be0e2f29b0"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

13 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

263

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 188
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "8251ea1b-2aa4-4e33-9d45-236b851177ae"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:10

13 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

140

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 157
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "5d5f4566-bcab-49ad-b1f3-d0255fe910bc"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:08

12 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.ReentrantLock$Sync.lockInterruptibly(ReentrantLock.java:159)
	at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:372)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly(SnapshotManagement.scala:86)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly$(SnapshotManagement.scala:85)
	at org.apache.spark.sql.delta.DeltaLog.withSnapshotLockInterruptibly(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$2(SnapshotManagement.scala:1092)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:171)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:169)
	at org.apache.spark.sql.delta.DeltaLog.recordFrameProfile(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:1091)
	at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:1059)
	at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at scala.Option.getOrElse(Option.scala:201)
	at org.apache.spark.sql.delta.OptimisticTransaction.<init>(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.DeltaLog.$anonfun$startTransaction$1(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.NoOpTransactionExecutionObserver$.startingTransaction(TransactionExecutionObserver.scala:109)
	at org.apache.spark.sql.delta.DeltaLog.startTransaction(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:246)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

195

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 54
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "a71ebdcc-81b1-4f0f-8ba8-05eb45a4bb71"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

12 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

213

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 90
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "14c8aa2c-e9b7-4654-8609-907121af310f"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

12 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'

222

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 108
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "821d0b0d-41ea-48a1-b8fa-a3c7cf284237"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:39

12 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

189

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 42
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "320e82c2-abf9-41e7-831c-5b4e4e6469f9"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

11 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

199

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 62
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "11505f12-b6ea-4784-8780-c85845c387a8"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

11 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

210

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 84
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "6bc41597-674a-4fee-9ab3-55277c54abe3"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:37

11 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

237

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 136
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "57ae2e76-b04c-45fc-bed4-787052fe1f0b"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:07

11 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

241

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 144
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e9d4efb1-75f2-497f-9d0f-d75b7833153b"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:08

11 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

246

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 154
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "2cd0c3f1-7085-448d-af0e-31d580b5d84f"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:08

11 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

262

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 186
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "131986c8-8cfc-4db3-a6cc-6201942f62e0"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:10

11 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { sql_command { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
      ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    sql_command {
      input {
        common {
          plan_id: 2
        }
        sql {
          query: "insert into yellow_taxi.trips\n        select \n            try_cast(VendorID as long),\n            try_cast(tpep_pickup_datetime as timestamp),\n            try_cast(tpep_dropoff_datetime as timestamp),\n            try_cast(passenger_count as long),\n            try_cast(trip_distance as double),\n            try_cast(RatecodeID as long),\n            try_cast(store_and_fwd_flag as string),\n            try_cast(PULocationID as long),\n            try_cast(DOLocationID as long),\n            try_cast(payment_type as long),\n            try_cast(fare_amount as double),\n            try_cast(extra as double),\n            try_cast(mta_tax as double),\n            try_cast(tip_amount as double),\n            try_cast(tolls_amount as double),\n            try_cast(improvement_surcharge as double),\n            try_cast(total_amount as double),\n            try_cast(congestion_surcharge as double),\n            try_cast(airport_fee as double),\n            current_timestamp,\n            coalesce(year(tpep_pickup_datetime),2026),\n  [truncated(size=1,230)]"
        }
      }
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "9343c9c1-8c6f-4a0b-ae77-3c533dca1d18"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 15:55:43

10 ms

TABLE_OR_VIEW_NOT_FOUND +details

[TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#21L as bigint) AS VendorID#42L, try_cast(tpep_pickup_datetime#22 as timestamp) AS tpep_pickup_datetime#43, try_cast(tpep_dropoff_datetime#23 as timestamp) AS tpep_dropoff_datetime#44, try_cast(passenger_count#24 as bigint) AS passenger_count#45L, try_cast(trip_distance#25 as double) AS trip_distance#46, try_cast(RatecodeID#26 as bigint) AS RatecodeID#47L, try_cast(store_and_fwd_flag#27 as string) AS store_and_fwd_flag#48, try_cast(PULocationID#28L as bigint) AS PULocationID#49L, try_cast(DOLocationID#29L as bigint) AS DOLocationID#50L, try_cast(payment_type#30L as bigint) AS payment_type#51L, try_cast(fare_amount#31 as double) AS fare_amount#52, try_cast(extra#32 as double) AS extra#53, try_cast(mta_tax#33 as double) AS mta_tax#54, try_cast(tip_amount#34 as double) AS tip_amount#55, try_cast(tolls_amount#35 as double) AS tolls_amount#56, try_cast(improvement_surcharge#36 as double) AS improvement_surcharge#57, try_cast(total_amount#37 as double) AS total_amount#58, try_cast(congestion_surcharge#38 as double) AS congestion_surcharge#59, try_cast(airport_fee#39 as double) AS airport_fee#60, current_timestamp() AS current_timestamp()#41, coalesce(year(cast(tpep_pickup_datetime#22 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#61, coalesce(month(cast(tpep_pickup_datetime#22 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#62, coalesce(day(cast(tpep_pickup_datetime#22 as date)), 3) AS coalesce(day(tpep_pickup_datetime), 3)#63]
   +- Relation [VendorID#21L,tpep_pickup_datetime#22,tpep_dropoff_datetime#23,passenger_count#24,trip_distance#25,RatecodeID#26,store_and_fwd_flag#27,PULocationID#28L,DOLocationID#29L,payment_type#30L,fare_amount#31,extra#32,mta_tax#33,tip_amount#34,tolls_amount#35,improvement_surcharge#36,total_amount#37,congestion_surcharge#38,airport_fee#39] parquet

127

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 137
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "8b26a669-e3a5-46d2-9682-646315f9fcf6"
client_observed_server_side_session_id: "944329a9-456e-41f7-bdab-39648b290e2e"

2026/09/03 16:10:06

10 ms

java.lang.InterruptedException +details

java.lang.InterruptedException
	at java.base/java.util.concurrent.locks.ReentrantLock$Sync.lockInterruptibly(ReentrantLock.java:159)
	at java.base/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:372)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly(SnapshotManagement.scala:86)
	at org.apache.spark.sql.delta.SnapshotManagement.withSnapshotLockInterruptibly$(SnapshotManagement.scala:85)
	at org.apache.spark.sql.delta.DeltaLog.withSnapshotLockInterruptibly(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.$anonfun$update$2(SnapshotManagement.scala:1092)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile(DeltaLogging.scala:171)
	at org.apache.spark.sql.delta.metering.DeltaLogging.recordFrameProfile$(DeltaLogging.scala:169)
	at org.apache.spark.sql.delta.DeltaLog.recordFrameProfile(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.SnapshotManagement.update(SnapshotManagement.scala:1091)
	at org.apache.spark.sql.delta.SnapshotManagement.update$(SnapshotManagement.scala:1059)
	at org.apache.spark.sql.delta.DeltaLog.update(DeltaLog.scala:78)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.OptimisticTransaction$$anonfun$$lessinit$greater$1.apply(OptimisticTransaction.scala:183)
	at scala.Option.getOrElse(Option.scala:201)
	at org.apache.spark.sql.delta.OptimisticTransaction.<init>(OptimisticTransaction.scala:183)
	at org.apache.spark.sql.delta.DeltaLog.$anonfun$startTransaction$1(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.NoOpTransactionExecutionObserver$.startingTransaction(TransactionExecutionObserver.scala:109)
	at org.apache.spark.sql.delta.DeltaLog.startTransaction(DeltaLog.scala:220)
	at org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:246)
	at org.apache.spark.sql.delta.commands.WriteIntoDelta.run(WriteIntoDelta.scala:105)
	at org.apache.spark.sql.delta.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:262)
	at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:55)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:79)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:77)
	at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:88)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$2(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$8(SQLExecution.scala:176)
	at org.apache.spark.sql.execution.SQLExecution$.withSessionTagsApplied(SQLExecution.scala:284)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$7(SQLExecution.scala:138)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$6(SQLExecution.scala:138)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:307)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId0$1(SQLExecution.scala:137)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId0(SQLExecution.scala:91)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:249)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$eagerlyExecuteCommands$1(QueryExecution.scala:185)
	at org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:717)
	at org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$eagerlyExecute$1(QueryExecution.scala:184)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:201)
	at org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$3.applyOrElse(QueryExecution.scala:194)
	at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:491)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:360)
	at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:356)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:37)
	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:467)
	at org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:194)
	at org.apache.spark.sql.execution.QueryExecution.$anonfun$lazyCommandExecuted$1(QueryExecution.scala:155)
	at scala.util.Try$.apply(Try.scala:217)
	at org.apache.spark.util.Utils$.doTryWithCallerStacktrace(Utils.scala:1392)
	at org.apache.spark.util.LazyTry.tryT$lzycompute(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.tryT(LazyTry.scala:46)
	at org.apache.spark.util.LazyTry.get(LazyTry.scala:58)
	at org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:160)
	at org.apache.spark.sql.execution.QueryExecution.assertCommandExecuted(QueryExecution.scala:239)
	at org.apache.spark.sql.connect.execution.SparkConnectPlanExecution.handlePlan(SparkConnectPlanExecution.scala:94)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1(ExecuteThreadRunner.scala:225)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.$anonfun$executeInternal$1$adapted(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$2(SessionHolder.scala:396)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:804)
	at org.apache.spark.sql.connect.service.SessionHolder.$anonfun$withSession$1(SessionHolder.scala:396)
	at org.apache.spark.JobArtifactSet$.withActiveJobArtifactState(JobArtifactSet.scala:94)
	at org.apache.spark.sql.artifact.ArtifactManager.$anonfun$withResources$1(ArtifactManager.scala:112)
	at org.apache.spark.util.Utils$.withContextClassLoader(Utils.scala:185)
	at org.apache.spark.sql.artifact.ArtifactManager.withClassLoaderIfNeeded(ArtifactManager.scala:102)
	at org.apache.spark.sql.artifact.ArtifactManager.withResources(ArtifactManager.scala:111)
	at org.apache.spark.sql.connect.service.SessionHolder.withSession(SessionHolder.scala:395)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.executeInternal(ExecuteThreadRunner.scala:197)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner.org$apache$spark$sql$connect$execution$ExecuteThreadRunner$$execute(ExecuteThreadRunner.scala:126)
	at org.apache.spark.sql.connect.execution.ExecuteThreadRunner$ExecutionThread.run(ExecuteThreadRunner.scala:334)

167

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 2
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "f1a22686-1c05-430b-bd21-db360e87205f"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:12:39

10 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

184

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 32
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "ed081ed8-4ab9-4e4d-a9d8-f25faeb6c9ed"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:34

10 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

193

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 50
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "453243e3-12f6-43ff-b295-28633ad30539"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

10 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

203

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 70
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "43d0c529-3218-4ae3-a405-0e00f1eb6600"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

10 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

217

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 98
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "15644fc0-5219-436f-978a-262f2851cff8"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

10 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

219

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 102
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "50badd40-73b6-413c-bf7f-913fa37c99a1"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

10 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

227

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 116
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "dc8a1ecd-9dcf-4368-b9f1-539b5cd080ee"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:06

10 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

239

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 140
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "8d6ace22-b5ab-4313-a159-d5f59b0f7b07"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:07

10 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

240

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 142
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "86f21334-439e-47d0-a875-fd51c6187d5c"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:08

10 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

252

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 166
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "c9f6afc9-1002-4e61-a0c6-75a6e13466d1"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:09

10 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

256

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 174
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "12b1ae76-d166-4c27-9c07-8187996ee91e"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:09

10 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'

169

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 6
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "49321596-692d-4bbb-9aa6-ac0b49869aae"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:12:39

9 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

178

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 20
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "993baa0e-ddcb-4699-b50a-50e6c4c21fde"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:33

9 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

179

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 22
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-12.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "070b7388-8ec5-4d50-b7fa-defa8b2fc8d7"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:33

9 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

182

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 28
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "b041af04-e936-43f4-ae5f-b9c73544bfdd"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:33

9 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

194

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 52
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-06.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

9 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

221

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 106
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-04.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "29dbea9c-4898-40e0-8360-9ebb3400571e"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:39

9 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'

229

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 120
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-11.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "581465f6-3066-4298-b9b9-5ec1adcbc575"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:06

9 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

231

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 124
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0ee9b11f-8e57-463c-86a9-723e65186262"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:06

9 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

243

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 148
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-03.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "88dfc4ad-bfe4-4e22-a465-84d5a27f6a7d"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:08

9 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

244

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 150
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-10.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "b5b19e34-af5f-43f4-8dae-a7a494428649"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:08

9 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

253

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 168
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "9d08ab95-a13f-4b32-af42-1a9b9a8db562"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:09

9 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'

180

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 24
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "f8ae7c29-a8ea-4205-a0a3-f1dd0000d6a4"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:33

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

183

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 30
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-07.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "8bcbc6a0-14e8-4a77-9380-2b94ecc9e95b"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:34

8 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

196

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 56
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "3a1b4a58-3756-4684-a865-1c18f8afce94"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:35

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

202

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 68
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "1a81ea2a-525d-4e8d-9b12-741b7aa7c3f9"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

204

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 72
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "4fae2a35-37ef-4ee1-a1b5-ff7757bf2cff"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

8 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

205

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 74
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "7338ba55-d94b-45e1-998b-8100ac497b10"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:36

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

212

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 88
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "1f032c4e-d4a8-4748-b54f-a53e0013de78"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:37

8 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

214

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 92
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "e22ae106-151d-445f-92a5-48264e354cc9"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

220

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 104
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-08.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0cc0f910-7a61-4785-aaf6-54f3223e0dab"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:13:38

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: void (nullable = true)

230

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 122
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-09.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "f1c57fb8-ee34-47e9-9a07-720cfd1c5e8e"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:06

8 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

232

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 126
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-02.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "d30ab6dc-8509-490d-9183-1f392faaeab6"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:07

8 ms

DELTA_FAILED_TO_MERGE_FIELDS +details

[DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'

233

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42" plan { command { write_operation { input { common { ... +details

Spark Connect - session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
  ...

session_id: "50cf85d5-ae27-4707-b30c-2f106b0fbf42"
plan {
  command {
    write_operation {
      input {
        common {
          plan_id: 128
        }
        read {
          data_source {
            format: "parquet"
            schema: ""
            paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-01.parquet"
          }
        }
      }
      source: "delta"
      path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
      mode: SAVE_MODE_APPEND
    }
  }
}
client_type: "_SPARK_CONNECT_PYTHON spark/4.1.0 os/linux python/3.13.11"
request_options {
  result_chunking_options {
    allow_arrow_batch_chunking: true
  }
}
request_options {
  reattach_options {
    reattachable: true
  }
}
operation_id: "0f6391dc-4186-438a-a313-ba7c32f0b0e6"
client_observed_server_side_session_id: "22ad9d42-0ffe-4297-9b48-76cbfeeba479"

2026/09/03 16:14:07

8 ms

DELTA_METADATA_MISMATCH +details

[DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.

Table schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: void (nullable = true)
 |-- airport_fee: void (nullable = true)


Data schema:
root
 |-- VendorID: long (nullable = true)
 |-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
 |-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
 |-- passenger_count: double (nullable = true)
 |-- trip_distance: double (nullable = true)
 |-- RatecodeID: double (nullable = true)
 |-- store_and_fwd_flag: string (nullable = true)
 |-- PULocationID: long (nullable = true)
 |-- DOLocationID: long (nullable = true)
 |-- payment_type: long (nullable = true)
 |-- fare_amount: double (nullable = true)
 |-- extra: double (nullable = true)
 |-- mta_tax: double (nullable = true)
 |-- tip_amount: double (nullable = true)
 |-- tolls_amount: double (nullable = true)
 |-- improvement_surcharge: double (nullable = true)
 |-- total_amount: double (nullable = true)
 |-- congestion_surcharge: double (nullable = true)
 |-- airport_fee: double (nullable = true)

Page: