fix(bulk_load): redownload file if local file damaged (#826)

XiaoMi · May 7, 2021 · 871f01f · 871f01f
1 parent 386a431
commit 871f01f
Showing 1 changed file with 25 additions and 6 deletions.
diff --git a/src/replica/bulk_load/replica_bulk_loader.cpp b/src/replica/bulk_load/replica_bulk_loader.cpp
@@ -415,15 +415,31 @@ error_code replica_bulk_loader::download_sst_files(const std::string &remote_dir
                     remote_dir, local_dir, f_meta.name, fs, f_size);
                 const std::string &file_name =
                     utils::filesystem::path_combine(local_dir, f_meta.name);
-                if (ec == ERR_OK || ec == ERR_PATH_ALREADY_EXIST) {
-                    if (!utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) {
-                        ec = ERR_CORRUPTION;
-                    } else if (ec == ERR_PATH_ALREADY_EXIST) {
+                bool verified = false;
+                if (ec == ERR_PATH_ALREADY_EXIST) {
+                    if (utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) {
                         // local file exist and is verified
                         ec = ERR_OK;
                         f_size = f_meta.size;
+                        verified = true;
+                    } else {
+                        derror_replica(
+                            "file({}) exists, but not verified, try to remove local file "
+                            "and redownload it",
+                            file_name);
+                        if (!utils::filesystem::remove_path(file_name)) {
+                            derror_replica("failed to remove file({})", file_name);
+                            ec = ERR_FILE_OPERATION_FAILED;
+                        } else {
+                            ec = _stub->_block_service_manager.download_file(
+                                remote_dir, local_dir, f_meta.name, fs, f_size);
+                        }
                     }
                 }
+                if (ec == ERR_OK && !verified &&
+                    !utils::filesystem::verify_file(file_name, f_meta.md5, f_meta.size)) {
+                    ec = ERR_CORRUPTION;
+                }
                 if (ec != ERR_OK) {
                     try_decrease_bulk_load_download_count();
                     _download_status.store(ec);
@@ -472,8 +488,11 @@ void replica_bulk_loader::update_bulk_load_download_progress(uint64_t file_size,
                                                              const std::string &file_name)
 {
     if (_metadata.file_total_size <= 0) {
-        derror_replica("bulk_load_metadata has invalid file_total_size({})",
-                       _metadata.file_total_size);
+        derror_replica("update downloading file({}) progress failed, metadata has invalid "
+                       "file_total_size({}), current status = {}",
+                       file_name,
+                       _metadata.file_total_size,
+                       enum_to_string(_status));
         return;
     }