Skip to content

Commit

Permalink
Remove usage of MicroPartition.cast_to_schema in read function
Browse files Browse the repository at this point in the history
  • Loading branch information
Jay Chia committed Feb 24, 2024
1 parent 82ad347 commit e8d6ba1
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 5 deletions.
6 changes: 2 additions & 4 deletions src/daft-micropartition/src/micropartition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ impl MicroPartition {
.map(|cols| cols.iter().map(|s| s.as_str()).collect::<Vec<&str>>());

let row_groups = parquet_sources_to_row_groups(scan_task.sources.as_slice());
let mp = read_parquet_into_micropartition(
read_parquet_into_micropartition(
uris.as_slice(),
columns.as_deref(),
None,
Expand All @@ -424,9 +424,7 @@ impl MicroPartition {
Some(schema.clone()),
field_id_mapping,
)
.context(DaftCoreComputeSnafu)?;

mp.cast_to_schema(schema).context(DaftCoreComputeSnafu)
.context(DaftCoreComputeSnafu)
}

// CASE: Last resort fallback option
Expand Down
4 changes: 3 additions & 1 deletion src/daft-scan/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,9 @@ impl DataFileSource {
pub struct ScanTask {
pub sources: Vec<DataFileSource>,

/// Schema to use when reading the DataFileSources.
/// Schema to use when reading the DataFileSources. This should always be passed in by the
/// ScanOperator implementation and should not have had any "pruning" applied.
///
/// Note that this is different than the schema of the data after pushdowns have been applied,
/// which can be obtained with [`ScanTask::materialized_schema`] instead.
pub schema: SchemaRef,
Expand Down

0 comments on commit e8d6ba1

Please sign in to comment.