From 9e513a75ea1a5e74662ca59deca9d152c9a78189 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Wed, 11 Sep 2024 21:48:10 +0200 Subject: [PATCH 1/5] Add arrow=53 --- Cargo.lock | 205 ++++++++++++-------- example/Cargo.toml | 6 +- integration_tests/Cargo.toml | 6 +- serde_arrow/Cargo.toml | 15 +- serde_arrow/benches/groups/impls.rs | 4 +- serde_arrow/benches/groups/json_to_arrow.rs | 6 +- serde_arrow/build.rs | 2 + serde_arrow/src/arrow_impl/array.rs | 16 +- serde_arrow/src/lib.rs | 2 + 9 files changed, 162 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb6f06a6..70601619 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,35 +76,35 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" +checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba" dependencies = [ "arrow-arith", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", "arrow-cast", "arrow-csv", - "arrow-data 52.0.0", + "arrow-data 53.0.0", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema 52.0.0", + "arrow-schema 53.0.0", "arrow-select", "arrow-string", ] [[package]] name = "arrow-arith" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" +checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "chrono", "half 2.2.1", "num", @@ -202,7 +202,7 @@ dependencies = [ "arrow-schema 42.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -218,7 +218,7 @@ dependencies = [ "arrow-schema 43.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -234,7 +234,7 @@ dependencies = [ "arrow-schema 44.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", "num-complex", ] @@ -251,7 +251,7 @@ dependencies = [ "arrow-schema 45.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -267,7 +267,7 @@ dependencies = [ "arrow-schema 46.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -283,7 +283,7 @@ dependencies = [ "arrow-schema 47.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -299,7 +299,7 @@ dependencies = [ "arrow-schema 48.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -315,7 +315,7 @@ dependencies = [ "arrow-schema 49.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -331,7 +331,7 @@ dependencies = [ "arrow-schema 50.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -347,7 +347,7 @@ dependencies = [ "arrow-schema 51.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", "num", ] @@ -363,7 +363,23 @@ dependencies = [ "arrow-schema 52.0.0", "chrono", "half 2.2.1", - "hashbrown 0.14.0", + "hashbrown 0.14.5", + "num", +] + +[[package]] +name = "arrow-array" +version = "53.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f" +dependencies = [ + "ahash 0.8.3", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", + "chrono", + "half 2.2.1", + "hashbrown 0.14.5", "num", ] @@ -534,16 +550,27 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "53.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d" +dependencies = [ + "bytes", + "half 2.2.1", + "num", +] + [[package]] name = "arrow-cast" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" +checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "arrow-select", "atoi", "base64", @@ -556,15 +583,15 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" +checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", "arrow-cast", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "chrono", "csv", "csv-core", @@ -765,31 +792,43 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-data" +version = "53.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65" +dependencies = [ + "arrow-buffer 53.0.0", + "arrow-schema 53.0.0", + "half 2.2.1", + "num", +] + [[package]] name = "arrow-ipc" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" +checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", "arrow-cast", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "flatbuffers", ] [[package]] name = "arrow-json" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" +checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", "arrow-cast", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "chrono", "half 2.2.1", "indexmap", @@ -801,14 +840,14 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" +checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "arrow-select", "half 2.2.1", "num", @@ -816,17 +855,16 @@ dependencies = [ [[package]] name = "arrow-row" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" +checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb" dependencies = [ "ahash 0.8.3", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "half 2.2.1", - "hashbrown 0.14.0", ] [[package]] @@ -973,30 +1011,39 @@ dependencies = [ "serde", ] +[[package]] +name = "arrow-schema" +version = "53.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858" +dependencies = [ + "serde", +] + [[package]] name = "arrow-select" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" +checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba" dependencies = [ "ahash 0.8.3", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "num", ] [[package]] name = "arrow-string" -version = "52.0.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" +checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array 53.0.0", + "arrow-buffer 53.0.0", + "arrow-data 53.0.0", + "arrow-schema 53.0.0", "arrow-select", "memchr", "num", @@ -1605,9 +1652,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hermit-abi" @@ -1646,7 +1693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.5", ] [[package]] @@ -1797,9 +1844,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "num" @@ -2248,6 +2295,7 @@ dependencies = [ "arrow-array 50.0.0", "arrow-array 51.0.0", "arrow-array 52.0.0", + "arrow-array 53.0.0", "arrow-buffer 37.0.0", "arrow-buffer 38.0.0", "arrow-buffer 39.0.0", @@ -2264,6 +2312,7 @@ dependencies = [ "arrow-buffer 50.0.0", "arrow-buffer 51.0.0", "arrow-buffer 52.0.0", + "arrow-buffer 53.0.0", "arrow-data 37.0.0", "arrow-data 38.0.0", "arrow-data 39.0.0", @@ -2280,6 +2329,7 @@ dependencies = [ "arrow-data 50.0.0", "arrow-data 51.0.0", "arrow-data 52.0.0", + "arrow-data 53.0.0", "arrow-json", "arrow-schema 37.0.0", "arrow-schema 38.0.0", @@ -2297,6 +2347,7 @@ dependencies = [ "arrow-schema 50.0.0", "arrow-schema 51.0.0", "arrow-schema 52.0.0", + "arrow-schema 53.0.0", "arrow2 0.16.0", "arrow2 0.17.0", "arrow2_convert", diff --git a/example/Cargo.toml b/example/Cargo.toml index 95731707..1b28ad77 100644 --- a/example/Cargo.toml +++ b/example/Cargo.toml @@ -7,11 +7,11 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -# arrow-version:replace: arrow = {{ version = "52.0", features = [{version}] }} -arrow = {version = "52.0", features = ["ipc"] } +# arrow-version:replace: arrow = {{ version = "{version}", features = ["ipc"] }} +arrow = {version = "53.0", features = ["ipc"] } chrono = { version = "0.4", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } # arrow-version:replace: serde_arrow = {{ path = "../serde_arrow", features = ["arrow-{version}"] }} -serde_arrow = { path = "../serde_arrow", features = ["arrow-52"] } +serde_arrow = { path = "../serde_arrow", features = ["arrow-53"] } diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index fd398ca3..a45707e9 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -4,12 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] -# arrow-version:replace: arrow = {{ version = "52.0", features = [{version}] }} -arrow = {version = "52.0", features = ["ipc"] } +# arrow-version:replace: arrow = {{ version = "{version}", features = ["ipc"] }} +arrow = {version = "53.0", features = ["ipc"] } chrono = { version = "0.4", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1" # arrow-version:replace: serde_arrow = {{ path = "../serde_arrow", features = ["arrow-{version}"] }} -serde_arrow = { path = "../serde_arrow", features = ["arrow-52"] } +serde_arrow = { path = "../serde_arrow", features = ["arrow-53"] } diff --git a/serde_arrow/Cargo.toml b/serde_arrow/Cargo.toml index a185071f..8ef43e3b 100644 --- a/serde_arrow/Cargo.toml +++ b/serde_arrow/Cargo.toml @@ -14,17 +14,18 @@ bench = false [[bench]] name = "serde_arrow_bench" # arrow-version:replace: required-features = ["arrow2-0-17", "arrow-{version}"] -required-features = ["arrow2-0-17", "arrow-52"] +required-features = ["arrow2-0-17", "arrow-53"] harness = false [package.metadata.docs.rs] # arrow-version:replace: features = ["arrow2-0-17", "arrow-{version}"] -features = ["arrow2-0-17", "arrow-52"] +features = ["arrow2-0-17", "arrow-53"] [features] default = [] # arrow-version:insert: arrow-{version} = ["dep:arrow-array-{version}", "dep:arrow-schema-{version}", "dep:arrow-data-{version}", "dep:arrow-buffer-{version}"] +arrow-53 = ["dep:arrow-array-53", "dep:arrow-schema-53", "dep:arrow-data-53", "dep:arrow-buffer-53"] arrow-52 = ["dep:arrow-array-52", "dep:arrow-schema-52", "dep:arrow-data-52", "dep:arrow-buffer-52"] arrow-51 = ["dep:arrow-array-51", "dep:arrow-schema-51", "dep:arrow-data-51", "dep:arrow-buffer-51"] arrow-50 = ["dep:arrow-array-50", "dep:arrow-schema-50", "dep:arrow-data-50", "dep:arrow-buffer-50"] @@ -50,6 +51,7 @@ half = { version = "2", features = ["bytemuck"], default-features = false } serde = { version = "1.0", features = ["derive", "std"], default-features = false } # arrow-version:insert: arrow-array-{version} = {{ package = "arrow-array", version = "{version}", optional = true, default-features = false }} +arrow-array-53 = { package = "arrow-array", version = "53", optional = true, default-features = false } arrow-array-52 = { package = "arrow-array", version = "52", optional = true, default-features = false } arrow-array-51 = { package = "arrow-array", version = "51", optional = true, default-features = false } arrow-array-50 = { package = "arrow-array", version = "50", optional = true, default-features = false } @@ -68,6 +70,7 @@ arrow-array-38 = { package = "arrow-array", version = "38", optional = true, def arrow-array-37 = { package = "arrow-array", version = "37", optional = true, default-features = false } # arrow-version:insert: arrow-buffer-{version} = {{ package = "arrow-buffer", version = "{version}", optional = true, default-features = false }} +arrow-buffer-53 = { package = "arrow-buffer", version = "53", optional = true, default-features = false } arrow-buffer-52 = { package = "arrow-buffer", version = "52", optional = true, default-features = false } arrow-buffer-51 = { package = "arrow-buffer", version = "51", optional = true, default-features = false } arrow-buffer-50 = { package = "arrow-buffer", version = "50", optional = true, default-features = false } @@ -86,6 +89,7 @@ arrow-buffer-38 = { package = "arrow-buffer", version = "38", optional = true, d arrow-buffer-37 = { package = "arrow-buffer", version = "37", optional = true, default-features = false } # arrow-version:insert: arrow-data-{version} = {{ package = "arrow-data", version="{version}", optional = true, default-features = false }} +arrow-data-53 = { package = "arrow-data", version="53", optional = true, default-features = false } arrow-data-52 = { package = "arrow-data", version="52", optional = true, default-features = false } arrow-data-51 = { package = "arrow-data", version="51", optional = true, default-features = false } arrow-data-50 = { package = "arrow-data", version="50", optional = true, default-features = false } @@ -104,6 +108,7 @@ arrow-data-38 = { package = "arrow-data", version="38", optional = true, default arrow-data-37 = { package = "arrow-data", version="37", optional = true, default-features = false } # arrow-version:insert: arrow-schema-{version} = {{ package = "arrow-schema", version = "{version}", optional = true, default-features = false }} +arrow-schema-53 = { package = "arrow-schema", version = "53", optional = true, default-features = false } arrow-schema-52 = { package = "arrow-schema", version = "52", optional = true, default-features = false } arrow-schema-51 = { package = "arrow-schema", version = "51", optional = true, default-features = false } arrow-schema-50 = { package = "arrow-schema", version = "50", optional = true, default-features = false } @@ -136,13 +141,14 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } # for benchmarks # arrow-version:replace: arrow-json-{version} = {{ package = "arrow-json", version = "{version}" }} -arrow-json-52 = { package = "arrow-json", version = "52" } +arrow-json-53 = { package = "arrow-json", version = "53" } criterion = "0.5" arrow2_convert = "0.5.0" serde-transcode = "1" simd-json = "0.13.8" # arrow-version:insert: arrow-schema-{version} = {{ package = "arrow-schema", version = "{version}", default-features = false, features = ["serde"] }} +arrow-schema-53 = { package = "arrow-schema", version = "53", default-features = false, features = ["serde"] } arrow-schema-52 = { package = "arrow-schema", version = "52", default-features = false, features = ["serde"] } arrow-schema-51 = { package = "arrow-schema", version = "51", default-features = false, features = ["serde"] } arrow-schema-50 = { package = "arrow-schema", version = "50", default-features = false, features = ["serde"] } @@ -178,6 +184,7 @@ check-cfg = [ 'cfg(has_arrow)', 'cfg(has_arrow_fixed_binary_support)', # arrow-version:insert: 'cfg(has_arrow_{version})', + 'cfg(has_arrow_53)', 'cfg(has_arrow_52)', 'cfg(has_arrow_51)', 'cfg(has_arrow_50)', @@ -194,4 +201,4 @@ check-cfg = [ 'cfg(has_arrow_39)', 'cfg(has_arrow_38)', 'cfg(has_arrow_37)', -] +] \ No newline at end of file diff --git a/serde_arrow/benches/groups/impls.rs b/serde_arrow/benches/groups/impls.rs index c8210479..6faac10b 100644 --- a/serde_arrow/benches/groups/impls.rs +++ b/serde_arrow/benches/groups/impls.rs @@ -125,9 +125,9 @@ pub mod arrow { use std::sync::Arc; // arrow-version:replace: use arrow_json_{version}::ReaderBuilder; - use arrow_json_52::ReaderBuilder; + use arrow_json_53::ReaderBuilder; // arrow-version:replace: use arrow_schema_{version}::Schema; - use arrow_schema_52::Schema; + use arrow_schema_53::Schema; use serde::Serialize; diff --git a/serde_arrow/benches/groups/json_to_arrow.rs b/serde_arrow/benches/groups/json_to_arrow.rs index f0fef921..063a92d2 100644 --- a/serde_arrow/benches/groups/json_to_arrow.rs +++ b/serde_arrow/benches/groups/json_to_arrow.rs @@ -6,13 +6,13 @@ use { }; // arrow-version:replace: use arrow_json_{version}::ReaderBuilder; -use arrow_json_52::ReaderBuilder; +use arrow_json_53::ReaderBuilder; // arrow-version:replace: use arrow_schema_{version}::{{FieldRef, Schema as ArrowSchema}}; -use arrow_schema_52::{FieldRef, Schema as ArrowSchema}; +use arrow_schema_53::{FieldRef, Schema as ArrowSchema}; // arrow-version:replace: use arrow_array_{version}::RecordBatch; -use arrow_array_52::RecordBatch; +use arrow_array_53::RecordBatch; use serde_json::Value; fn benchmark_json_to_arrow(c: &mut criterion::Criterion) { diff --git a/serde_arrow/build.rs b/serde_arrow/build.rs index 7c5b9639..2f2b7f7c 100644 --- a/serde_arrow/build.rs +++ b/serde_arrow/build.rs @@ -15,6 +15,8 @@ fn main() { let max_arrow_version: Option = [ // arrow-version:insert: #[cfg(feature = "arrow-{version}")]{\n}{version}, + #[cfg(feature = "arrow-53")] + 53, #[cfg(feature = "arrow-52")] 52, #[cfg(feature = "arrow-51")] diff --git a/serde_arrow/src/arrow_impl/array.rs b/serde_arrow/src/arrow_impl/array.rs index 9c03b3dd..a256ca3e 100644 --- a/serde_arrow/src/arrow_impl/array.rs +++ b/serde_arrow/src/arrow_impl/array.rs @@ -51,7 +51,7 @@ impl TryFrom for ArrayData { T::Boolean, // NOTE: use the explicit len arr.len, - arr.validity.map(Buffer::from), + arr.validity.map(Buffer::from_vec), 0, vec![ScalarBuffer::from(arr.values).into_inner()], vec![], @@ -114,7 +114,7 @@ impl TryFrom for ArrayData { Ok(ArrayData::builder(data_type) .len(arr.len) - .null_bit_buffer(arr.validity.map(Buffer::from)) + .null_bit_buffer(arr.validity.map(Buffer::from_vec)) .child_data(data) .build()?) } @@ -153,7 +153,7 @@ impl TryFrom for ArrayData { Ok(ArrayData::try_new( T::FixedSizeList(Arc::new(field), arr.n), child.len() / usize::try_from(arr.n)?, - arr.validity.map(Buffer::from), + arr.validity.map(Buffer::from_vec), 0, vec![], vec![child], @@ -170,7 +170,7 @@ impl TryFrom for ArrayData { Ok(ArrayData::try_new( T::FixedSizeBinary(arr.n), arr.data.len() / usize::try_from(arr.n)?, - arr.validity.map(Buffer::from), + arr.validity.map(Buffer::from_vec), 0, vec![ScalarBuffer::from(arr.data).into_inner()], vec![], @@ -196,7 +196,7 @@ impl TryFrom for ArrayData { Ok(ArrayData::try_new( T::Map(Arc::new(field), false), arr.offsets.len().saturating_sub(1), - arr.validity.map(Buffer::from), + arr.validity.map(Buffer::from_vec), 0, vec![ScalarBuffer::from(arr.offsets).into_inner()], vec![child], @@ -543,7 +543,7 @@ fn primitive_into_data( Ok(ArrayData::try_new( data_type, values.len(), - validity.map(Buffer::from), + validity.map(Buffer::from_vec), 0, vec![ScalarBuffer::from(values).into_inner()], vec![], @@ -559,7 +559,7 @@ fn bytes_into_data( Ok(ArrayData::try_new( data_type, offsets.len().saturating_sub(1), - validity.map(Buffer::from), + validity.map(Buffer::from_vec), 0, vec![ ScalarBuffer::from(offsets).into_inner(), @@ -579,7 +579,7 @@ fn list_into_data( Ok(ArrayData::try_new( data_type, len, - validity.map(Buffer::from), + validity.map(Buffer::from_vec), 0, vec![ScalarBuffer::from(offsets).into_inner()], vec![child_data], diff --git a/serde_arrow/src/lib.rs b/serde_arrow/src/lib.rs index 1ce4d392..16a3cbcb 100644 --- a/serde_arrow/src/lib.rs +++ b/serde_arrow/src/lib.rs @@ -145,6 +145,7 @@ //! | Arrow Feature | Arrow Version | //! |---------------|---------------| // arrow-version:insert: //! | `arrow-{version}` | `arrow={version}` | +//! | `arrow-53` | `arrow=53` | //! | `arrow-52` | `arrow=52` | //! | `arrow-51` | `arrow=51` | //! | `arrow-50` | `arrow=50` | @@ -268,6 +269,7 @@ pub mod _impl { } // arrow-version:insert: #[cfg(has_arrow_{version})] build_arrow_crate!(arrow_array_{version}, arrow_buffer_{version}, arrow_data_{version}, arrow_schema_{version}); +#[cfg(has_arrow_53)] build_arrow_crate!(arrow_array_53, arrow_buffer_53, arrow_data_53, arrow_schema_53); #[cfg(has_arrow_52)] build_arrow_crate!(arrow_array_52, arrow_buffer_52, arrow_data_52, arrow_schema_52); #[cfg(has_arrow_51)] build_arrow_crate!(arrow_array_51, arrow_buffer_51, arrow_data_51, arrow_schema_51); #[cfg(has_arrow_50)] build_arrow_crate!(arrow_array_50, arrow_buffer_50, arrow_data_50, arrow_schema_50); From b9ec6bfc8bee9c657a951d4cafeafe44e44e6083 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Wed, 11 Sep 2024 21:48:19 +0200 Subject: [PATCH 2/5] Fix x.py add-arrow-version --- x.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x.py b/x.py index df6d900e..144ddc5e 100644 --- a/x.py +++ b/x.py @@ -7,6 +7,7 @@ all_arrow_features = [ # arrow-version:insert: "arrow-{version}", + "arrow-53", "arrow-52", "arrow-51", "arrow-50", @@ -538,8 +539,8 @@ def add_arrow_version(version): for p in [ self_path / "x.py", - *self_path.glob("serde_arrow/**/*.rs"), - *self_path.glob("serde_arrow/**/*.toml"), + *self_path.glob("*/**/*.rs"), + *self_path.glob("*/**/*.toml"), ]: content = p.read_text() if "arrow-version" not in content: From 5688bdb0f4a7fcc1aeba0a5aeb496cefca5c1e6d Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Wed, 11 Sep 2024 21:48:24 +0200 Subject: [PATCH 3/5] Fix clippy --- serde_arrow/src/arrow_impl/schema.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/serde_arrow/src/arrow_impl/schema.rs b/serde_arrow/src/arrow_impl/schema.rs index 338e51dc..064e291d 100644 --- a/serde_arrow/src/arrow_impl/schema.rs +++ b/serde_arrow/src/arrow_impl/schema.rs @@ -141,13 +141,13 @@ impl TryFrom<&ArrowDataType> for DataType { AT::Date32 => Ok(T::Date32), AT::Date64 => Ok(T::Date64), AT::Decimal128(precision, scale) => Ok(T::Decimal128(*precision, *scale)), - AT::Time32(unit) => Ok(T::Time32(unit.clone().into())), - AT::Time64(unit) => Ok(T::Time64(unit.clone().into())), + AT::Time32(unit) => Ok(T::Time32((*unit).into())), + AT::Time64(unit) => Ok(T::Time64((*unit).into())), AT::Timestamp(unit, tz) => Ok(T::Timestamp( - unit.clone().into(), + (*unit).into(), tz.as_ref().map(|s| s.to_string()), )), - AT::Duration(unit) => Ok(T::Duration(unit.clone().into())), + AT::Duration(unit) => Ok(T::Duration((*unit).into())), AT::Binary => Ok(T::Binary), AT::LargeBinary => Ok(T::LargeBinary), AT::FixedSizeBinary(n) => Ok(T::FixedSizeBinary(*n)), From e5ee1e16fce6b592d2138c91cd80db7a8cca3c29 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Wed, 11 Sep 2024 21:58:40 +0200 Subject: [PATCH 4/5] Revert Copy change to support previous Arrow versions --- serde_arrow/src/arrow_impl/schema.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/serde_arrow/src/arrow_impl/schema.rs b/serde_arrow/src/arrow_impl/schema.rs index 064e291d..2e3da229 100644 --- a/serde_arrow/src/arrow_impl/schema.rs +++ b/serde_arrow/src/arrow_impl/schema.rs @@ -141,13 +141,27 @@ impl TryFrom<&ArrowDataType> for DataType { AT::Date32 => Ok(T::Date32), AT::Date64 => Ok(T::Date64), AT::Decimal128(precision, scale) => Ok(T::Decimal128(*precision, *scale)), - AT::Time32(unit) => Ok(T::Time32((*unit).into())), - AT::Time64(unit) => Ok(T::Time64((*unit).into())), + AT::Time32(unit) => Ok(T::Time32( + // only some arrow version implement Copy for unit + #[allow(clippy::clone_on_copy)] + unit.clone().into(), + )), + AT::Time64(unit) => Ok(T::Time64( + // only some arrow version implement Copy for unit + #[allow(clippy::clone_on_copy)] + unit.clone().into(), + )), AT::Timestamp(unit, tz) => Ok(T::Timestamp( - (*unit).into(), + // only some arrow version implement Copy for unit + #[allow(clippy::clone_on_copy)] + unit.clone().into(), tz.as_ref().map(|s| s.to_string()), )), - AT::Duration(unit) => Ok(T::Duration((*unit).into())), + AT::Duration(unit) => Ok(T::Duration( + // only some arrow version implement Copy for unit + #[allow(clippy::clone_on_copy)] + unit.clone().into(), + )), AT::Binary => Ok(T::Binary), AT::LargeBinary => Ok(T::LargeBinary), AT::FixedSizeBinary(n) => Ok(T::FixedSizeBinary(*n)), From 2f57a6fd8ec8074be301de10be77f332f62c4b31 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Wed, 11 Sep 2024 22:04:03 +0200 Subject: [PATCH 5/5] Update changelog --- Changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.md b/Changes.md index 03c3664c..aca50d64 100644 --- a/Changes.md +++ b/Changes.md @@ -6,6 +6,7 @@ Refactor the underlying implementation to prepare for further development New features +- Add `arrow=53` support - Add `Binary` / `LargeBinary` support for `arrow2` - Add support to serialize / deserialize `bool` from integer arrays - Add a helper to construct `Bool8` arrays