From eeefc4728f91725aa1f6e5042ecf67550414ad83 Mon Sep 17 00:00:00 2001 From: ritchie Date: Wed, 31 Jan 2024 12:30:39 +0100 Subject: [PATCH 1/2] fix: don't see files as hive partitions --- crates/polars-plan/src/logical_plan/hive.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/polars-plan/src/logical_plan/hive.rs b/crates/polars-plan/src/logical_plan/hive.rs index d95d50907ecf..4d34eb0a47de 100644 --- a/crates/polars-plan/src/logical_plan/hive.rs +++ b/crates/polars-plan/src/logical_plan/hive.rs @@ -47,6 +47,14 @@ impl HivePartitions { let name = it.next()?; let value = it.next()?; + // Don't see files `foo=1.parquet` as hive partitions. + // So we return globs and paths with extensions. + if value.contains('*') { + return None; + } + let value_path = Path::new(value); + let _ = value_path.extension()?; + // Having multiple '=' doesn't seem like valid hive partition, // continue as url. if it.next().is_some() { From f9cb14a9afc0540859358710cd93e5fbfcc733bc Mon Sep 17 00:00:00 2001 From: ritchie Date: Wed, 31 Jan 2024 13:49:02 +0100 Subject: [PATCH 2/2] reverse --- crates/polars-plan/src/logical_plan/hive.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/polars-plan/src/logical_plan/hive.rs b/crates/polars-plan/src/logical_plan/hive.rs index 4d34eb0a47de..c22784e505d5 100644 --- a/crates/polars-plan/src/logical_plan/hive.rs +++ b/crates/polars-plan/src/logical_plan/hive.rs @@ -53,7 +53,9 @@ impl HivePartitions { return None; } let value_path = Path::new(value); - let _ = value_path.extension()?; + if value_path.extension().is_some() { + return None; + } // Having multiple '=' doesn't seem like valid hive partition, // continue as url.