From 0c01c0e688037d855a87d70b6b7025d91d62c077 Mon Sep 17 00:00:00 2001 From: Mustafa Akur Date: Sat, 30 Nov 2024 10:37:46 -0800 Subject: [PATCH] Initial commit --- datafusion-cli/Cargo.lock | 4 ---- datafusion/common/src/cse.rs | 2 +- datafusion/common/src/lib.rs | 4 ++++ datafusion/expr/Cargo.toml | 1 - datafusion/expr/src/logical_plan/builder.rs | 3 +-- datafusion/expr/src/logical_plan/plan.rs | 3 +-- datafusion/expr/src/utils.rs | 3 +-- datafusion/optimizer/src/eliminate_duplicated_expr.rs | 3 +-- datafusion/optimizer/src/push_down_filter.rs | 3 +-- .../optimizer/src/simplify_expressions/expr_simplifier.rs | 5 +++-- datafusion/physical-expr/Cargo.toml | 1 - datafusion/physical-expr/src/equivalence/properties.rs | 5 +++-- datafusion/physical-expr/src/window/window_expr.rs | 4 +--- datafusion/physical-plan/Cargo.toml | 1 - .../physical-plan/src/windows/bounded_window_agg_exec.rs | 3 +-- datafusion/sql/Cargo.toml | 1 - datafusion/sql/src/select.rs | 3 +-- datafusion/sql/src/unparser/utils.rs | 3 +-- datafusion/sql/src/utils.rs | 6 ++---- 19 files changed, 22 insertions(+), 36 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 04b0b0d22cfd..765d6b5bb0f2 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -1349,7 +1349,6 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap", "paste", "recursive", "serde_json", @@ -1511,7 +1510,6 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap", "itertools", "log", "paste", @@ -1567,7 +1565,6 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap", "itertools", "log", "once_cell", @@ -1586,7 +1583,6 @@ dependencies = [ "arrow-schema", "datafusion-common", "datafusion-expr", - "indexmap", "log", "recursive", "regex", diff --git a/datafusion/common/src/cse.rs b/datafusion/common/src/cse.rs index ab02915858cd..bb9abd539908 100644 --- a/datafusion/common/src/cse.rs +++ b/datafusion/common/src/cse.rs @@ -24,8 +24,8 @@ use crate::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter, TreeNodeVisitor, }; +use crate::IndexMap; use crate::Result; -use indexmap::IndexMap; use std::collections::HashMap; use std::hash::{BuildHasher, Hash, Hasher, RandomState}; use std::marker::PhantomData; diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 77e8cd60ede2..ad3b6ab0d171 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -73,6 +73,7 @@ pub use param_value::ParamValues; pub use scalar::{ScalarType, ScalarValue}; pub use schema_reference::SchemaReference; pub use stats::{ColumnStatistics, Statistics}; +use std::hash::RandomState; pub use table_reference::{ResolvedTableReference, TableReference}; pub use unnest::{RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; @@ -93,6 +94,9 @@ pub use error::{ pub type HashMap = hashbrown::HashMap; pub type HashSet = hashbrown::HashSet; +pub type IndexMap = indexmap::IndexMap; +pub type IndexSet = indexmap::IndexSet; + /// Downcast an Arrow Array to a concrete type, return an `DataFusionError::Internal` if the cast is /// not possible. In normal usage of DataFusion the downcast should always succeed. /// diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 438662e0642b..772a57ac93d3 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -49,7 +49,6 @@ datafusion-expr-common = { workspace = true } datafusion-functions-aggregate-common = { workspace = true } datafusion-functions-window-common = { workspace = true } datafusion-physical-expr-common = { workspace = true } -indexmap = { workspace = true } paste = "^1.0" recursive = { workspace = true } serde_json = { workspace = true } diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index 90235e3f84c4..27143b0db35e 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -54,11 +54,10 @@ use datafusion_common::file_options::file_type::FileType; use datafusion_common::{ exec_err, get_target_functional_dependencies, internal_err, not_impl_err, plan_datafusion_err, plan_err, Column, DFSchema, DFSchemaRef, DataFusionError, - FunctionalDependencies, Result, ScalarValue, TableReference, ToDFSchema, + FunctionalDependencies, IndexSet, Result, ScalarValue, TableReference, ToDFSchema, UnnestOptions, }; use datafusion_expr_common::type_coercion::binary::type_union_resolution; -use indexmap::IndexSet; /// Default table name for unnamed table pub const UNNAMED_TABLE: &str = "?table?"; diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index e2590a9f255c..96786c69a375 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -51,10 +51,9 @@ use datafusion_common::tree_node::{ use datafusion_common::{ aggregate_functional_dependencies, internal_err, plan_err, Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency, FunctionalDependence, - FunctionalDependencies, ParamValues, Result, ScalarValue, TableReference, + FunctionalDependencies, IndexSet, ParamValues, Result, ScalarValue, TableReference, UnnestOptions, }; -use indexmap::IndexSet; // backwards compatibility use crate::display::PgJsonVisitor; diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index 6f7c5d379260..b99b9e7f4c1c 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -36,10 +36,9 @@ use datafusion_common::tree_node::{ use datafusion_common::utils::get_at_indices; use datafusion_common::{ internal_err, plan_datafusion_err, plan_err, Column, DFSchema, DFSchemaRef, - DataFusionError, HashMap, Result, TableReference, + DataFusionError, HashMap, IndexSet, Result, TableReference, }; -use indexmap::IndexSet; use sqlparser::ast::{ExceptSelectItem, ExcludeSelectItem}; pub use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity; diff --git a/datafusion/optimizer/src/eliminate_duplicated_expr.rs b/datafusion/optimizer/src/eliminate_duplicated_expr.rs index 554985667fdf..54acaa335f5d 100644 --- a/datafusion/optimizer/src/eliminate_duplicated_expr.rs +++ b/datafusion/optimizer/src/eliminate_duplicated_expr.rs @@ -20,10 +20,9 @@ use crate::optimizer::ApplyOrder; use crate::{OptimizerConfig, OptimizerRule}; use datafusion_common::tree_node::Transformed; -use datafusion_common::Result; +use datafusion_common::{IndexSet, Result}; use datafusion_expr::logical_plan::LogicalPlan; use datafusion_expr::{Aggregate, Expr, Sort, SortExpr}; -use indexmap::IndexSet; use std::hash::{Hash, Hasher}; /// Optimization rule that eliminate duplicated expr. #[derive(Default, Debug)] diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index fe751a5fb583..955da36cfa01 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -17,7 +17,6 @@ //! [`PushDownFilter`] applies filters as early as possible -use indexmap::IndexSet; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -27,7 +26,7 @@ use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, }; use datafusion_common::{ - internal_err, plan_err, qualified_name, Column, DFSchema, Result, + internal_err, plan_err, qualified_name, Column, DFSchema, IndexSet, Result, }; use datafusion_expr::expr_rewriter::replace_col; use datafusion_expr::logical_plan::{Join, JoinType, LogicalPlan, TableScan, Union}; diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 60de3f6b3673..bb67dea008a6 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -31,7 +31,9 @@ use datafusion_common::{ cast::{as_large_list_array, as_list_array}, tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter}, }; -use datafusion_common::{internal_err, DFSchema, DataFusionError, Result, ScalarValue}; +use datafusion_common::{ + internal_err, DFSchema, DataFusionError, IndexSet, Result, ScalarValue, +}; use datafusion_expr::simplify::ExprSimplifyResult; use datafusion_expr::{ and, lit, or, BinaryExpr, Case, ColumnarValue, Expr, Like, Operator, Volatility, @@ -43,7 +45,6 @@ use datafusion_expr::{ utils::{iter_conjunction, iter_conjunction_owned}, }; use datafusion_physical_expr::{create_physical_expr, execution_props::ExecutionProps}; -use indexmap::IndexSet; use crate::analyzer::type_coercion::TypeCoercionRewriter; use crate::simplify_expressions::guarantees::GuaranteeRewriter; diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index db3e0e10d816..d00067e3befb 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -48,7 +48,6 @@ datafusion-functions-aggregate-common = { workspace = true } datafusion-physical-expr-common = { workspace = true } half = { workspace = true } hashbrown = { workspace = true } -indexmap = { workspace = true } itertools = { workspace = true, features = ["use_std"] } log = { workspace = true } paste = "^1.0" diff --git a/datafusion/physical-expr/src/equivalence/properties.rs b/datafusion/physical-expr/src/equivalence/properties.rs index fe866450b2b2..6c049196c4d7 100644 --- a/datafusion/physical-expr/src/equivalence/properties.rs +++ b/datafusion/physical-expr/src/equivalence/properties.rs @@ -36,12 +36,13 @@ use crate::{ use arrow_schema::{SchemaRef, SortOptions}; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; -use datafusion_common::{internal_err, plan_err, JoinSide, JoinType, Result}; +use datafusion_common::{ + internal_err, plan_err, IndexMap, IndexSet, JoinSide, JoinType, Result, +}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_physical_expr_common::utils::ExprPropertiesNode; -use indexmap::{IndexMap, IndexSet}; use itertools::Itertools; /// A `EquivalenceProperties` object stores information known about the output diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index 8b130506cdea..3070eb2b3f3d 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -28,14 +28,12 @@ use arrow::compute::SortOptions; use arrow::datatypes::Field; use arrow::record_batch::RecordBatch; use datafusion_common::utils::compare_rows; -use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, DataFusionError, IndexMap, Result, ScalarValue}; use datafusion_expr::window_state::{ PartitionBatchState, WindowAggState, WindowFrameContext, WindowFrameStateGroups, }; use datafusion_expr::{Accumulator, PartitionEvaluator, WindowFrame, WindowFrameBound}; -use indexmap::IndexMap; - /// Common trait for [window function] implementations /// /// # Aggregate Window Expressions diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index bb0e21fdfd15..7dd69e69eb39 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -57,7 +57,6 @@ datafusion-physical-expr-common = { workspace = true } futures = { workspace = true } half = { workspace = true } hashbrown = { workspace = true } -indexmap = { workspace = true } itertools = { workspace = true, features = ["use_std"] } log = { workspace = true } once_cell = "1.18.0" diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs index 398b5eb292d7..287ef6d767ef 100644 --- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs @@ -52,7 +52,7 @@ use datafusion_common::utils::{ evaluate_partition_ranges, get_at_indices, get_row_at_idx, }; use datafusion_common::{ - arrow_datafusion_err, exec_err, DataFusionError, HashMap, Result, + arrow_datafusion_err, exec_err, DataFusionError, HashMap, IndexMap, Result, }; use datafusion_execution::TaskContext; use datafusion_expr::window_state::{PartitionBatchState, WindowAggState}; @@ -65,7 +65,6 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement}; use futures::stream::Stream; use futures::{ready, StreamExt}; use hashbrown::raw::RawTable; -use indexmap::IndexMap; use log::debug; /// Window execution plan diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index 01bf92043816..b3a28fc9d561 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -46,7 +46,6 @@ arrow-array = { workspace = true } arrow-schema = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } -indexmap = { workspace = true } log = { workspace = true } recursive = { workspace = true } regex = { workspace = true } diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 39b6eb6e8132..002185adafd6 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -25,7 +25,7 @@ use crate::utils::{ }; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; -use datafusion_common::{not_impl_err, plan_err, Result}; +use datafusion_common::{not_impl_err, plan_err, IndexMap, Result}; use datafusion_common::{RecursionUnnestOption, UnnestOptions}; use datafusion_expr::expr::{Alias, PlannedReplaceSelectItem, WildcardOptions}; use datafusion_expr::expr_rewriter::{ @@ -38,7 +38,6 @@ use datafusion_expr::{ qualified_wildcard_with_options, wildcard_with_options, Aggregate, Expr, Filter, GroupingSet, LogicalPlan, LogicalPlanBuilder, Partitioning, }; -use indexmap::IndexMap; use sqlparser::ast::{ Distinct, Expr as SQLExpr, GroupByExpr, NamedWindowExpr, OrderByExpr, WildcardAdditionalOptions, WindowType, diff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rs index c77af7924384..c5aed2929e50 100644 --- a/datafusion/sql/src/unparser/utils.rs +++ b/datafusion/sql/src/unparser/utils.rs @@ -20,13 +20,12 @@ use std::{cmp::Ordering, sync::Arc, vec}; use datafusion_common::{ internal_err, tree_node::{Transformed, TransformedResult, TreeNode}, - Column, DataFusionError, Result, ScalarValue, + Column, DataFusionError, IndexSet, Result, ScalarValue, }; use datafusion_expr::{ expr, utils::grouping_set_to_exprlist, Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, Projection, SortExpr, Unnest, Window, }; -use indexmap::IndexSet; use sqlparser::ast; use super::{ diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index e29a2299b41c..3d028d71f749 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -27,7 +27,7 @@ use datafusion_common::tree_node::{ }; use datafusion_common::{ exec_err, internal_err, plan_err, Column, DFSchemaRef, DataFusionError, HashMap, - Result, ScalarValue, + IndexMap, Result, ScalarValue, }; use datafusion_expr::builder::get_struct_unnested_columns; use datafusion_expr::expr::{Alias, GroupingSet, Unnest, WindowFunction}; @@ -35,7 +35,6 @@ use datafusion_expr::utils::{expr_as_column_expr, find_column_exprs}; use datafusion_expr::{ col, expr_vec_fmt, ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, }; -use indexmap::IndexMap; use sqlparser::ast::{Ident, Value}; /// Make a best-effort attempt at resolving all columns in the expression tree @@ -618,13 +617,12 @@ mod tests { use arrow::datatypes::{DataType as ArrowDataType, Field, Schema}; use arrow_schema::Fields; - use datafusion_common::{Column, DFSchema, Result}; + use datafusion_common::{Column, DFSchema, IndexMap, Result}; use datafusion_expr::{ col, lit, unnest, ColumnUnnestList, EmptyRelation, LogicalPlan, }; use datafusion_functions::core::expr_ext::FieldAccessor; use datafusion_functions_aggregate::expr_fn::count; - use indexmap::IndexMap; use crate::utils::{resolve_positions_to_exprs, rewrite_recursive_unnest_bottom_up};