Update DuckDB to ^v0.8.0 (#92)

* update duckdb * remove experimental setting which is now a default * update read csv to expect headers and comma delim duckdb 0.8.x is now more permissive of csv reads and now requires further specificity in order to emit exceptions with csv's that have values but include missing headers.
cytomining · Aug 24, 2023 · 29464c1 · 29464c1
1 parent 0563891
commit 29464c1
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 59 deletions.
diff --git a/cytotable/convert.py b/cytotable/convert.py
@@ -216,7 +216,7 @@ def _get_table_chunk_offsets(
             _duckdb_reader()
             .execute(
                 # nosec
-                f"SELECT COUNT(*) from read_csv_auto('{source_path}')"
+                f"SELECT COUNT(*) from read_csv_auto('{source_path}', header=TRUE, delim=',')"
                 if source_type == ".csv"
                 else f"SELECT COUNT(*) from sqlite_scan('{source_path}', '{table_name}')"
             )
@@ -299,7 +299,7 @@ def _source_chunk_to_parquet(
     # build output query and filepath base
     # (chunked output will append offset to keep output paths unique)
     if str(AnyPath(source["source_path"]).suffix).lower() == ".csv":
-        base_query = f"SELECT {select_columns} FROM read_csv_auto('{str(source['source_path'])}')"
+        base_query = f"SELECT {select_columns} FROM read_csv_auto('{str(source['source_path'])}', header=TRUE, delim=',')"
         result_filepath_base = f"{source_dest_path}/{str(source['source_path'].stem)}"
 
     elif str(AnyPath(source["source_path"]).suffix).lower() == ".sqlite":

diff --git a/cytotable/utils.py b/cytotable/utils.py
@@ -192,13 +192,6 @@ def _duckdb_reader() -> duckdb.DuckDBPyConnection:
         https://duckdb.org/docs/sql/configuration#configuration-reference
         */
         PRAGMA preserve_insertion_order=FALSE;
-
-        /*
-        Allow parallel csv reads for performance increase possibilities
-        See the following for more information:
-        https://duckdb.org/docs/sql/configuration#configuration-reference
-        */
-        PRAGMA experimental_parallel_csv=TRUE;
         """,
     )
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -10,7 +10,7 @@ packages = [{include = "cytotable"}]
 python = ">=3.8,<3.13"
 pyarrow = "^12.0.0"
 cloudpathlib = {extras = ["all"], version = "^0.13.0"}
-duckdb = "^0.7.0"
+duckdb = "^0.8.0"
 parsl = "^2023.4.24"
 
 [tool.poetry.dev-dependencies]