Skip to content

Commit

Permalink
clean up of python rewrite
Browse files Browse the repository at this point in the history
  • Loading branch information
samster25 committed Dec 16, 2023
1 parent 62a8332 commit b6c4a59
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 14 deletions.
3 changes: 2 additions & 1 deletion daft/iceberg/iceberg_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ def partitioning_keys(self) -> list[PartitionField]:

def _iceberg_record_to_partition_spec(self, record: Record) -> daft.table.Table | None:
arrays = dict()
assert len(record._position_to_field_name) == len(self._partition_keys)
for name, value, pfield in zip(record._position_to_field_name, record.record_fields(), self._partition_keys):
field = Field._from_pyfield(pfield.field)
field_name = field.name
Expand All @@ -103,7 +104,7 @@ def to_scan_tasks(self, pushdowns: Pushdowns) -> Iterator[ScanTask]:
limit = pushdowns.limit
iceberg_tasks = self._table.scan(limit=limit).plan_files()

limit_files = limit is not None and pushdowns.filters is None
limit_files = limit is not None and pushdowns.filters is None and pushdowns.partition_filters is None

scan_tasks = []

Expand Down
2 changes: 1 addition & 1 deletion src/daft-scan/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ pub struct Pushdowns {

impl Default for Pushdowns {
fn default() -> Self {
Self::new(None, None, None, None)
Self::new(None, None, None, None)
}
}

Expand Down
12 changes: 0 additions & 12 deletions src/daft-scan/src/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -214,18 +214,6 @@ partitioning_keys:\n",
) -> common_error::DaftResult<
Box<dyn Iterator<Item = common_error::DaftResult<crate::ScanTaskRef>>>,
> {
println!("{:?}", pushdowns.filters);
if let Some(pred) = pushdowns.filters.as_deref() {
let transformed = rewrite_predicate_for_partitioning(
pred.as_ref().clone(),
self.partitioning_keys.as_slice(),
)?;
println!("before {}", pred);
for t in transformed {
println!(" {t}");
}
}

let scan_tasks = Python::with_gil(|py| {
let pypd = PyPushdowns(pushdowns.into()).into_py(py);
let pyiter =
Expand Down

0 comments on commit b6c4a59

Please sign in to comment.