fix(core): Fix failed list related tests (#5058)

* fix(core): Fix failed list related tests Signed-off-by: Xuanwo <github@xuanwo.io> * Fix build Signed-off-by: Xuanwo <github@xuanwo.io> * Fix azdls Signed-off-by: Xuanwo <github@xuanwo.io> * Fix b2 Signed-off-by: Xuanwo <github@xuanwo.io> * Fix b2 Signed-off-by: Xuanwo <github@xuanwo.io> * Fix aliyun drive Signed-off-by: Xuanwo <github@xuanwo.io> * Fix aliyun drive Signed-off-by: Xuanwo <github@xuanwo.io> --------- Signed-off-by: Xuanwo <github@xuanwo.io>
apache · Aug 27, 2024 · 8bc883f · 8bc883f
1 parent 69ed149
commit 8bc883f
Show file tree

Hide file tree

Showing 8 changed files with 62 additions and 72 deletions.
diff --git a/core/src/raw/oio/entry.rs b/core/src/raw/oio/entry.rs
@@ -38,7 +38,12 @@ impl Entry {
     }
 
     /// Create a new entry with given value.
-    pub fn with(path: String, meta: Metadata) -> Entry {
+    pub fn with(mut path: String, meta: Metadata) -> Entry {
+        // Normalize path as `/` if it's empty.
+        if path.is_empty() {
+            path = "/".to_string();
+        }
+
         debug_assert!(
             meta.mode().is_dir() == path.ends_with('/'),
             "mode {:?} not match with path {}",

diff --git a/core/src/services/aliyun_drive/lister.rs b/core/src/services/aliyun_drive/lister.rs
@@ -65,26 +65,19 @@ impl oio::PageList for AliyunDriveLister {
         };
 
         let offset = if ctx.token.is_empty() {
-            if !parent.path.ends_with('/') {
-                // List "dir" should contains "dir/".
-                let path = if !parent.path.starts_with('/') {
-                    format!("/{}", parent.path)
-                } else {
-                    parent.path.clone()
-                };
-                ctx.entries.push_back(Entry::new(
-                    &format!("{}/", path),
-                    Metadata::new(EntryMode::DIR).with_last_modified(
-                        parent
-                            .updated_at
-                            .parse::<chrono::DateTime<Utc>>()
-                            .map_err(|e| {
-                                Error::new(ErrorKind::Unexpected, "parse last modified time")
-                                    .set_source(e)
-                            })?,
-                    ),
-                ));
-            }
+            // Push self into the list result.
+            ctx.entries.push_back(Entry::new(
+                &parent.path,
+                Metadata::new(EntryMode::DIR).with_last_modified(
+                    parent
+                        .updated_at
+                        .parse::<chrono::DateTime<Utc>>()
+                        .map_err(|e| {
+                            Error::new(ErrorKind::Unexpected, "parse last modified time")
+                                .set_source(e)
+                        })?,
+                ),
+            ));
             None
         } else {
             Some(ctx.token.clone())
@@ -107,57 +100,35 @@ impl oio::PageList for AliyunDriveLister {
         let result: AliyunDriveFileList =
             serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
 
-        let n = result.items.len();
-
         for item in result.items {
-            let path = if parent.path.starts_with('/') {
-                build_abs_path(&parent.path, &item.name)
+            let (path, mut md) = if item.path_type == "folder" {
+                let path = format!("{}{}/", &parent.path.trim_start_matches('/'), &item.name);
+                (path, Metadata::new(EntryMode::DIR))
             } else {
-                build_abs_path(&format!("/{}", &parent.path), &item.name)
+                let path = format!("{}{}", &parent.path.trim_start_matches('/'), &item.name);
+                (path, Metadata::new(EntryMode::FILE))
             };
 
-            let (path, md) = if item.path_type == "folder" {
-                let path = format!("{}/", path);
-                let meta = Metadata::new(EntryMode::DIR).with_last_modified(
-                    item.updated_at
-                        .parse::<chrono::DateTime<Utc>>()
-                        .map_err(|e| {
-                            Error::new(ErrorKind::Unexpected, "parse last modified time")
-                                .set_source(e)
-                        })?,
-                );
-                (path, meta)
-            } else {
-                let mut meta = Metadata::new(EntryMode::FILE).with_last_modified(
-                    item.updated_at
-                        .parse::<chrono::DateTime<Utc>>()
-                        .map_err(|e| {
-                            Error::new(ErrorKind::Unexpected, "parse last modified time")
-                                .set_source(e)
-                        })?,
-                );
-                if let Some(v) = item.size {
-                    meta = meta.with_content_length(v);
-                }
-                if let Some(v) = item.content_type {
-                    meta = meta.with_content_type(v);
-                }
-                (path, meta)
-            };
+            md = md.with_last_modified(item.updated_at.parse::<chrono::DateTime<Utc>>().map_err(
+                |e| Error::new(ErrorKind::Unexpected, "parse last modified time").set_source(e),
+            )?);
+            if let Some(v) = item.size {
+                md = md.with_content_length(v);
+            }
+            if let Some(v) = item.content_type {
+                md = md.with_content_type(v);
+            }
 
             ctx.entries.push_back(Entry::new(&path, md));
         }
 
-        if self.limit.is_some_and(|x| x < n) || result.next_marker.is_none() {
+        let next_marker = result.next_marker.unwrap_or_default();
+        if next_marker.is_empty() {
             ctx.done = true;
+        } else {
+            ctx.token = next_marker;
         }
 
-        if let Some(marker) = result.next_marker {
-            if marker.is_empty() {
-                ctx.done = true;
-            }
-            ctx.token = marker;
-        }
         Ok(())
     }
 }
diff --git a/core/src/services/azdls/lister.rs b/core/src/services/azdls/lister.rs
@@ -56,6 +56,12 @@ impl oio::PageList for AzdlsLister {
             return Err(parse_error(resp).await?);
         }
 
+        // Return self at the first page.
+        if ctx.token.is_empty() && !ctx.done {
+            let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR));
+            ctx.entries.push_back(e);
+        }
+
         // Check whether this list is done.
         if let Some(value) = resp.headers().get("x-ms-continuation") {
             let value = value.to_str().map_err(|err| {
@@ -90,7 +96,7 @@ impl oio::PageList for AzdlsLister {
                 .with_last_modified(parse_datetime_from_rfc2822(&object.last_modified)?);
 
             let mut path = build_rel_path(&self.core.root, &object.name);
-            if mode == EntryMode::DIR {
+            if mode.is_dir() {
                 path += "/"
             };
 

diff --git a/core/src/services/azfile/lister.rs b/core/src/services/azfile/lister.rs
@@ -56,6 +56,12 @@ impl oio::PageList for AzfileLister {
             return Err(parse_error(resp).await?);
         }
 
+        // Return self at the first page.
+        if ctx.token.is_empty() && !ctx.done {
+            let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR));
+            ctx.entries.push_back(e);
+        }
+
         let bs = resp.into_body();
 
         let results: EnumerationResults =

diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs
@@ -96,9 +96,6 @@ impl oio::PageList for B2Lister {
                     continue;
                 }
             }
-            if file.file_name == build_abs_path(&self.core.root, &self.path) {
-                continue;
-            }
             let file_name = file.file_name.clone();
             let metadata = parse_file_info(&file);
 

diff --git a/core/src/services/cos/lister.rs b/core/src/services/cos/lister.rs
@@ -81,10 +81,9 @@ impl oio::PageList for CosLister {
         }
 
         for object in output.contents {
-            let path = build_rel_path(&self.core.root, &object.key);
-
-            if path == self.path || path.is_empty() {
-                continue;
+            let mut path = build_rel_path(&self.core.root, &object.key);
+            if path.is_empty() {
+                path = "/".to_string();
             }
 
             let meta = Metadata::new(EntryMode::from_path(&path)).with_content_length(object.size);

diff --git a/core/src/services/gdrive/lister.rs b/core/src/services/gdrive/lister.rs
@@ -64,6 +64,13 @@ impl oio::PageList for GdriveLister {
             return Ok(());
         }
 
+        // Return self at the first page.
+        if ctx.token.is_empty() && !ctx.done {
+            let path = build_rel_path(&self.core.root, &self.path);
+            let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR));
+            ctx.entries.push_back(e);
+        }
+
         let decoded_response =
             serde_json::from_slice::<GdriveFileList>(&bytes).map_err(new_json_deserialize_error)?;
 

diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs
@@ -251,11 +251,10 @@ impl kv::Adapter for Adapter {
         let pool = self.get_client().await?;
 
         let value = sqlx::query_scalar(&format!(
-            "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1 and `{}` <> $2",
-            self.key_field, self.table, self.key_field, self.key_field
+            "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1",
+            self.key_field, self.table, self.key_field
         ))
         .bind(format!("{path}%"))
-        .bind(path)
         .fetch_all(pool)
         .await
         .map_err(parse_sqlite_error)?;