Update arrow 48.0.0

Author: tustvold

Cargo.toml

@@ -48,12 +48,12 @@ rust-version = "1.70"
 version = "32.0.0"
 
 [workspace.dependencies]
-arrow = { version = "47.0.0", features = ["prettyprint"] }
-arrow-array = { version = "47.0.0", default-features = false, features = ["chrono-tz"] }
-arrow-buffer = { version = "47.0.0", default-features = false }
-arrow-flight = { version = "47.0.0", features = ["flight-sql-experimental"] }
-arrow-schema = { version = "47.0.0", default-features = false }
-parquet = { version = "47.0.0", features = ["arrow", "async", "object_store"] }
+arrow = { version = "48.0.0", features = ["prettyprint"] }
+arrow-array = { version = "48.0.0", default-features = false, features = ["chrono-tz"] }
+arrow-buffer = { version = "48.0.0", default-features = false }
+arrow-flight = { version = "48.0.0", features = ["flight-sql-experimental"] }
+arrow-schema = { version = "48.0.0", default-features = false }
+parquet = { version = "48.0.0", features = ["arrow", "async", "object_store"] }
 sqlparser = { version = "0.38.0", features = ["visitor"] }
 chrono = { version = "0.4.31", default-features = false }
 
@@ -74,3 +74,11 @@ opt-level = 3
 overflow-checks = false
 panic = 'unwind'
 rpath = false
+
+[patch.crates-io]
+arrow = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-array = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-buffer = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-flight = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-schema = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+parquet = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }

datafusion-cli/Cargo.lock

@@ -29,7 +29,7 @@ rust-version = "1.70"
 readme = "README.md"
 
 [dependencies]
-arrow = "47.0.0"
+arrow = "48.0.0"
 async-trait = "0.1.41"
 aws-config = "0.55"
 aws-credential-types = "0.55"
@@ -50,3 +50,10 @@ assert_cmd = "2.0"
 ctor = "0.2.0"
 predicates = "3.0"
 rstest = "0.17"
+
+[patch.crates-io]
+arrow = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-array = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-buffer = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+arrow-schema = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }
+parquet = { git = "https://github.com/tustvold/arrow-rs.git", rev = "27c795d5e4dc2351eb57f7225cd9dd051d3651c3" }

datafusion-cli/Cargo.toml

@@ -50,7 +50,7 @@ half = { version = "2.1", default-features = false }
 num_cpus = "1.13.0"
 object_store = { version = "0.7.0", default-features = false, optional = true }
 parquet = { workspace = true, optional = true }
-pyo3 = { version = "0.19.0", optional = true }
+pyo3 = { version = "0.20.0", optional = true }
 sqlparser = { workspace = true }
 
 [dev-dependencies]

datafusion/common/Cargo.toml

@@ -37,13 +37,6 @@ pub struct CsvWriterOptions {
     /// Compression to apply after ArrowWriter serializes RecordBatches.
     /// This compression is applied by DataFusion not the ArrowWriter itself.
     pub compression: CompressionTypeVariant,
-    /// Indicates whether WriterBuilder.has_header() is set to true.
-    /// This is duplicative as WriterBuilder also stores this information.
-    /// However, WriterBuilder does not allow public read access to the
-    /// has_header parameter.
-    pub has_header: bool,
-    // TODO: expose a way to read has_header in arrow create
-    // https://github.com/apache/arrow-rs/issues/4735
 }
 
 impl CsvWriterOptions {
@@ -54,7 +47,6 @@ impl CsvWriterOptions {
         Self {
             writer_options,
             compression,
-            has_header: true,
         }
     }
 }
@@ -65,29 +57,20 @@ impl TryFrom<(&ConfigOptions, &StatementOptions)> for CsvWriterOptions {
     fn try_from(value: (&ConfigOptions, &StatementOptions)) -> Result<Self> {
         let _configs = value.0;
         let statement_options = value.1;
-        let mut has_header = true;
         let mut builder = WriterBuilder::default();
         let mut compression = CompressionTypeVariant::UNCOMPRESSED;
         for (option, value) in &statement_options.options {
             builder = match option.to_lowercase().as_str(){
                 "header" => {
-                    has_header = value.parse()
+                    let has_header = value.parse()
                         .map_err(|_| DataFusionError::Configuration(format!("Unable to parse {value} as bool as required for {option}!")))?;
-                    builder.has_headers(has_header)
+                    builder.with_header(has_header)
                 },
                 "date_format" => builder.with_date_format(value.to_owned()),
                 "datetime_format" => builder.with_datetime_format(value.to_owned()),
                 "timestamp_format" => builder.with_timestamp_format(value.to_owned()),
                 "time_format" => builder.with_time_format(value.to_owned()),
-                "rfc3339" => {
-                    let value_bool = value.parse()
-                        .map_err(|_| DataFusionError::Configuration(format!("Unable to parse {value} as bool as required for {option}!")))?;
-                    if value_bool{
-                        builder.with_rfc3339()
-                    } else{
-                        builder
-                    }
-                },
+                "rfc3339" => builder, // No-op
                 "null_value" => builder.with_null(value.to_owned()),
                 "compression" => {
                     compression = CompressionTypeVariant::from_str(value.replace('\'', "").as_str())?;
@@ -112,7 +95,6 @@ impl TryFrom<(&ConfigOptions, &StatementOptions)> for CsvWriterOptions {
             }
         }
         Ok(CsvWriterOptions {
-            has_header,
             writer_options: builder,
             compression,
         })