From d87b53a2b8be8b0f69b433c0a7351892b4032b7a Mon Sep 17 00:00:00 2001 From: Keeyou Date: Tue, 5 Dec 2023 12:42:34 +0800 Subject: [PATCH 1/4] core: reimplment RemoveFile --- src/core/utils_fs.cpp | 66 ++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/src/core/utils_fs.cpp b/src/core/utils_fs.cpp index 2670e002f..1e35f22ef 100644 --- a/src/core/utils_fs.cpp +++ b/src/core/utils_fs.cpp @@ -3,10 +3,13 @@ #include "core/utils.hpp" #include "core/utils_fs.hpp" -#if !defined(_WIN32_WINNT) || _WIN32_WINNT >= 0x601 #include -#else + +#ifdef _WIN32 #include +#else +#include +#include #endif namespace yass { @@ -53,17 +56,6 @@ bool CreateDirectories(const std::string& path) { } return true; } - -bool RemoveFile(const std::string& path) { - std::error_code ec; - std::filesystem::path p(path); - std::filesystem::remove(p, ec); - if (ec) { - return false; - } - return true; -} - #else bool IsFile(const std::string& path) { @@ -94,15 +86,55 @@ bool CreateDirectories(const std::string& path) { } return true; } +#endif +#ifndef _WIN32 bool RemoveFile(const std::string& path) { - std::wstring wpath = SysUTF8ToWide(path); - if (::DeleteFileW(wpath.c_str()) == FALSE) { - return false; + const char* path_str = path.c_str(); + struct stat file_info {}; + if (::lstat(path_str, &file_info) != 0) { + // The Windows version defines this condition as success. + return (errno == ENOENT); } - return true; + if (S_ISDIR(file_info.st_mode)) { + return (rmdir(path_str) == 0) || (errno == ENOENT); + } else { + return (unlink(path_str) == 0) || (errno == ENOENT); + } +} +#else +// Returns the Win32 last error code or ERROR_SUCCESS if the last error code is +// ERROR_FILE_NOT_FOUND or ERROR_PATH_NOT_FOUND. This is useful in cases where +// the absence of a file or path is a success condition (e.g., when attempting +// to delete an item in the filesystem). +static DWORD ReturnLastErrorOrSuccessOnNotFound() { + const DWORD error_code = ::GetLastError(); + return (error_code == ERROR_FILE_NOT_FOUND || + error_code == ERROR_PATH_NOT_FOUND) + ? ERROR_SUCCESS + : error_code; } +bool RemoveFile(const std::string& path) { + std::wstring wpath = SysUTF8ToWide(path); + const DWORD attr = ::GetFileAttributesW(wpath.c_str()); + if (attr == INVALID_FILE_ATTRIBUTES) + return ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS; + // Clear the read-only bit if it is set. + if ((attr & FILE_ATTRIBUTE_READONLY) && + !::SetFileAttributesW(wpath.c_str(), + attr & ~DWORD{FILE_ATTRIBUTE_READONLY})) { + // It's possible for |path| to be gone now under a race with other deleters. + return ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS; + } + if (attr & FILE_ATTRIBUTE_DIRECTORY) { + return ::RemoveDirectoryW(wpath.c_str()) == TRUE ? true : + ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS; + } else { + return ::DeleteFileW(wpath.c_str()) == TRUE ? true : + ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS; + } +} #endif } // yass namespace From 9e6ce87fed153895483c0f13c22010a8d28d80c2 Mon Sep 17 00:00:00 2001 From: Keeyou Date: Tue, 5 Dec 2023 12:05:17 +0800 Subject: [PATCH 2/4] core: add read file and write file test --- src/core/utils.cpp | 20 +++++++++++++++++++- src/core/utils_test.cpp | 30 ++++++++++++++++++++++++++++++ third_party/libc++/CMakeLists.txt | 4 ++-- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/core/utils.cpp b/src/core/utils.cpp index d3726ac16..e2270e08b 100644 --- a/src/core/utils.cpp +++ b/src/core/utils.cpp @@ -272,6 +272,22 @@ ssize_t ReadFileToBuffer(const std::string& path, char* buf, size_t buf_len) { return ret; } +static bool WriteFileDescriptor(int fd, std::string_view data) { + // Allow for partial writes. + ssize_t bytes_written_total = 0; + ssize_t size = static_cast(data.size()); + DCHECK_LE(data.size(), static_cast(SSIZE_MAX)); // checked_cast + for (ssize_t bytes_written_partial = 0; bytes_written_total < size; + bytes_written_total += bytes_written_partial) { + bytes_written_partial = + HANDLE_EINTR(::write(fd, data.data() + bytes_written_total, + static_cast(size - bytes_written_total))); + if (bytes_written_partial < 0) + return false; + } + return true; +} + ssize_t WriteFileWithBuffer(const std::string& path, const char* buf, size_t buf_len) { @@ -280,7 +296,9 @@ ssize_t WriteFileWithBuffer(const std::string& path, if (fd < 0) { return false; } - ssize_t ret = HANDLE_EINTR(::write(fd, buf, buf_len)); + + ssize_t ret = WriteFileDescriptor(fd, std::string_view(buf, buf_len)) + ? buf_len : -1; if (IGNORE_EINTR(close(fd)) < 0) { return -1; diff --git a/src/core/utils_test.cpp b/src/core/utils_test.cpp index f08558e9c..9799762a9 100644 --- a/src/core/utils_test.cpp +++ b/src/core/utils_test.cpp @@ -3,12 +3,20 @@ #include +#include "core/process_utils.hpp" +#include "core/rand_util.hpp" #include "core/utils.hpp" +#include "core/utils_fs.hpp" #ifdef _WIN32 #include #endif +#include +#include + +using namespace yass; + TEST(UtilsTest, Dirname) { ASSERT_EQ(Dirname("a/b/prog/file.cc"), "a/b/prog"); ASSERT_EQ(Dirname("a/b/prog//"), "a/b"); @@ -85,3 +93,25 @@ TEST(UtilsTest, StringToInteger) { i = StringToInteger(std::string(s4, 4)); ASSERT_FALSE(i.ok()) << i.status(); } + +TEST(UtilsTest, ReadFileAndWrite4K) { + std::string buf, buf2; + buf.resize(4096); + buf2.resize(4096); + RandBytes(buf.data(), buf.size()); + int tmp_suffix; + RandBytes(&tmp_suffix, sizeof(tmp_suffix)); + auto tmp_name = absl::StrFormat("read_write_file-%u-%d", GetPID(), tmp_suffix); + auto tmp_dir = std::filesystem::path(::testing::TempDir()); +#ifdef _WIN32 + std::string tmp = SysWideToUTF8(tmp_dir / tmp_name); +#else + std::string tmp = tmp_dir / tmp_name; +#endif + + ASSERT_TRUE(WriteFileWithBuffer(tmp, buf.c_str(), buf.size())); + ASSERT_TRUE(ReadFileToBuffer(tmp, buf2.data(), buf2.size()+1)); + ASSERT_EQ(buf, buf2); + + ASSERT_TRUE(RemoveFile(tmp)); +} diff --git a/third_party/libc++/CMakeLists.txt b/third_party/libc++/CMakeLists.txt index da1a4d261..f6fe03577 100644 --- a/third_party/libc++/CMakeLists.txt +++ b/third_party/libc++/CMakeLists.txt @@ -237,6 +237,8 @@ set(libcxx_SOURCES trunk/src/valarray.cpp trunk/src/vector.cpp trunk/src/verbose_abort.cpp + trunk/src/filesystem/path.cpp + trunk/src/filesystem/filesystem_error.cpp ) # TODO move to base @@ -244,9 +246,7 @@ if (NOT WIN32 OR NOT ALLOW_XP) set(libcxx_SOURCES ${libcxx_SOURCES} trunk/src/filesystem/directory_iterator.cpp - trunk/src/filesystem/filesystem_error.cpp trunk/src/filesystem/operations.cpp - trunk/src/filesystem/path.cpp ) endif() From cdf0932210c700fd23e470ccb13f3781b0ed3822 Mon Sep 17 00:00:00 2001 From: Keeyou Date: Tue, 5 Dec 2023 13:18:42 +0800 Subject: [PATCH 3/4] core: implement GetTempDir --- src/core/utils.cpp | 18 ++++++++++++++++++ src/core/utils.hpp | 5 +++++ src/core/utils_mac.mm | 15 +++++++++++++++ src/core/utils_test.cpp | 8 ++++++++ src/core/utils_win.cpp | 30 ++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+) diff --git a/src/core/utils.cpp b/src/core/utils.cpp index e2270e08b..2c3ce4da2 100644 --- a/src/core/utils.cpp +++ b/src/core/utils.cpp @@ -225,6 +225,24 @@ void SetExecutablePath(const std::string& exe_path) { GetExecutablePath(&new_exe_path); absl::flags_internal::SetProgramInvocationName(new_exe_path); } + +bool GetTempDir(std::string *path) { + const char* tmp = getenv("TMPDIR"); + if (tmp) { + *path = tmp; + return true; + } +#if 0 // BUILDFLAG(IS_ANDROID) + return PathService::Get(DIR_CACHE, path); +#else +#if defined(__ANDROID__) + *path = "/data/local/tmp"; +#else + *path = "/tmp"; +#endif + return true; +#endif +} #endif /* diff --git a/src/core/utils.hpp b/src/core/utils.hpp index 7e76f1d6b..c30242450 100644 --- a/src/core/utils.hpp +++ b/src/core/utils.hpp @@ -155,6 +155,11 @@ void SetExecutablePath(const std::string& exe_path); void SetExecutablePath(const std::wstring& exe_path); #endif +bool GetTempDir(std::string *path); +#ifdef _WIN32 +bool GetTempDir(std::wstring *path); +#endif + bool Net_ipv6works(); #ifdef _MSC_VER diff --git a/src/core/utils_mac.mm b/src/core/utils_mac.mm index 9ad4de7d5..5fa26eb8a 100644 --- a/src/core/utils_mac.mm +++ b/src/core/utils_mac.mm @@ -9,6 +9,7 @@ #include #include +#import #include #include @@ -22,6 +23,7 @@ #include // For mlock. #include +#include #include "core/logging.hpp" #ifndef VM_MEMORY_MALLOC_PROB_GUARD @@ -528,4 +530,17 @@ void SetExecutablePath(const std::string& exe_path) { absl::flags_internal::SetProgramInvocationName(new_exe_path); } +bool GetTempDir(std::string *path) { + const char* env_tmpdir = getenv("TMPDIR"); + if (env_tmpdir) { + *path = env_tmpdir; + return true; + } + NSString* tmp = NSTemporaryDirectory(); + if (tmp == nil) { + return false; + } + *path = gurl_base::SysNSStringToUTF8(tmp); + return true; +} #endif // __APPLE__ diff --git a/src/core/utils_test.cpp b/src/core/utils_test.cpp index 9799762a9..12ddd6dfe 100644 --- a/src/core/utils_test.cpp +++ b/src/core/utils_test.cpp @@ -3,6 +3,7 @@ #include +#include "core/logging.hpp" #include "core/process_utils.hpp" #include "core/rand_util.hpp" #include "core/utils.hpp" @@ -94,6 +95,13 @@ TEST(UtilsTest, StringToInteger) { ASSERT_FALSE(i.ok()) << i.status(); } +TEST(UtilsTest, GetTempDir) { + std::string tmp_dir; + ASSERT_TRUE(GetTempDir(&tmp_dir)); + ASSERT_FALSE(tmp_dir.empty()); + LOG(ERROR) << "tmp_dir: " << tmp_dir; +} + TEST(UtilsTest, ReadFileAndWrite4K) { std::string buf, buf2; buf.resize(4096); diff --git a/src/core/utils_win.cpp b/src/core/utils_win.cpp index 837ef8c27..dfeb71907 100644 --- a/src/core/utils_win.cpp +++ b/src/core/utils_win.cpp @@ -710,6 +710,36 @@ void SetExecutablePath(const std::wstring& exe_path) { absl::flags_internal::SetProgramInvocationName(new_exe_path); } +bool GetTempDir(std::string *path) { + std::wstring wpath; + path->clear(); + if (!GetTempDir(&wpath)) { + return false; + } + *path = SysWideToUTF8(wpath); + return true; +} + +bool GetTempDir(std::wstring *path) { + wchar_t temp_path[MAX_PATH + 1]; + DWORD path_len = ::GetTempPathW(MAX_PATH, temp_path); + // If the function succeeds, the return value is the length, + // in TCHARs, of the string copied to lpBuffer, + // not including the terminating null character. + if (path_len >= MAX_PATH || path_len <= 0) + return false; + // TODO(evanm): the old behavior of this function was to always strip the + // trailing slash. We duplicate this here, but it shouldn't be necessary + // when everyone is using the appropriate FilePath APIs. + if (temp_path[path_len-1] == L'\\') { + temp_path[path_len-1] = L'\0'; + --path_len; + } + *path = std::wstring(temp_path, path_len); + DCHECK_NE((*path)[path_len-1], L'\0'); + return true; +} + ssize_t ReadFileToBuffer(const std::string& path, char* buf, size_t buf_len) { DWORD read; HANDLE hFile = ::CreateFileW(SysUTF8ToWide(path).c_str(), GENERIC_READ, From 9b930697b323acb3bdaf1978fcbb08f6d45d5d04 Mon Sep 17 00:00:00 2001 From: Keeyou Date: Tue, 5 Dec 2023 13:36:30 +0800 Subject: [PATCH 4/4] core: implement GetHomeDir --- CMakeLists.txt | 4 ++-- src/core/utils.cpp | 17 +++++++++++++++++ src/core/utils.hpp | 5 +++++ src/core/utils_mac.mm | 17 +++++++++++++++++ src/core/utils_test.cpp | 6 ++++++ src/core/utils_win.cpp | 26 ++++++++++++++++++++++++-- 6 files changed, 71 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e5e3e60eb..5c3024e6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1677,9 +1677,9 @@ if (WIN32) set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE) if (MSVC) - set(SYSTEM_LIBS ${SYSTEM_LIBS} Crypt32 WS2_32 bcrypt User32 kernel32) + set(SYSTEM_LIBS ${SYSTEM_LIBS} Crypt32 WS2_32 bcrypt User32 shell32 kernel32) else() - set(SYSTEM_LIBS ${SYSTEM_LIBS} crypt32 ws2_32 mswsock bcrypt user32 kernel32) + set(SYSTEM_LIBS ${SYSTEM_LIBS} crypt32 ws2_32 mswsock bcrypt user32 shell32 kernel32) endif() # Don't use it even if we are compiling with gnu extensions diff --git a/src/core/utils.cpp b/src/core/utils.cpp index 2c3ce4da2..4948cfaed 100644 --- a/src/core/utils.cpp +++ b/src/core/utils.cpp @@ -243,6 +243,23 @@ bool GetTempDir(std::string *path) { return true; #endif } + +std::string GetHomeDir() { + const char* home_dir = getenv("HOME"); + if (home_dir && home_dir[0]) { + return home_dir; + } +#if defined(__ANDROID__) + DLOG(WARNING) << "OS_ANDROID: Home directory lookup not yet implemented."; +#endif + // Fall back on temp dir if no home directory is defined. + std::string rv; + if (GetTempDir(&rv)) { + return rv; + } + // Last resort. + return "/tmp"; +} #endif /* diff --git a/src/core/utils.hpp b/src/core/utils.hpp index c30242450..0c854bb74 100644 --- a/src/core/utils.hpp +++ b/src/core/utils.hpp @@ -160,6 +160,11 @@ bool GetTempDir(std::string *path); bool GetTempDir(std::wstring *path); #endif +std::string GetHomeDir(); +#ifdef _WIN32 +std::wstring GetHomeDirW(); +#endif + bool Net_ipv6works(); #ifdef _MSC_VER diff --git a/src/core/utils_mac.mm b/src/core/utils_mac.mm index 5fa26eb8a..14295e7b7 100644 --- a/src/core/utils_mac.mm +++ b/src/core/utils_mac.mm @@ -543,4 +543,21 @@ bool GetTempDir(std::string *path) { *path = gurl_base::SysNSStringToUTF8(tmp); return true; } + +std::string GetHomeDir() { + NSString* tmp = NSHomeDirectory(); + if (tmp != nil) { + auto path = gurl_base::SysNSStringToUTF8(tmp); + if (!path.empty()) { + return path; + } + } + // Fall back on temp dir if no home directory is defined. + std::string rv; + if (GetTempDir(&rv)) { + return rv; + } + // Last resort. + return "/tmp"; +} #endif // __APPLE__ diff --git a/src/core/utils_test.cpp b/src/core/utils_test.cpp index 12ddd6dfe..5ead97a35 100644 --- a/src/core/utils_test.cpp +++ b/src/core/utils_test.cpp @@ -102,6 +102,12 @@ TEST(UtilsTest, GetTempDir) { LOG(ERROR) << "tmp_dir: " << tmp_dir; } +TEST(UtilsTest, GetHomeDir) { + std::string home_dir = GetHomeDir(); + ASSERT_FALSE(home_dir.empty()); + LOG(ERROR) << "home_dir: " << home_dir; +} + TEST(UtilsTest, ReadFileAndWrite4K) { std::string buf, buf2; buf.resize(4096); diff --git a/src/core/utils_win.cpp b/src/core/utils_win.cpp index dfeb71907..0dc087995 100644 --- a/src/core/utils_win.cpp +++ b/src/core/utils_win.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2022 Chilledheart */ +#ifdef _WIN32 // We use dynamic loading for below functions #define GetProductInfo GetProductInfoHidden @@ -10,9 +11,11 @@ #include "core/compiler_specific.hpp" #include "core/logging.hpp" -#include +#include +#include +#include -#ifdef _WIN32 +#include #define MAKE_WIN_VER(major, minor, build_number) \ (((major) << 24) | ((minor) << 16) | (build_number)) @@ -740,6 +743,25 @@ bool GetTempDir(std::wstring *path) { return true; } +std::string GetHomeDir() { + return SysWideToUTF8(GetHomeDirW()); +} + +std::wstring GetHomeDirW() { + wchar_t result[MAX_PATH]; + if (SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_PROFILE, NULL, SHGFP_TYPE_CURRENT, + result)) && result[0]) { + return result; + } + // Fall back to the temporary directory on failure. + std::wstring rv; + if (GetTempDir(&rv)) { + return rv; + } + // Last resort. + return L"C:\\"; +} + ssize_t ReadFileToBuffer(const std::string& path, char* buf, size_t buf_len) { DWORD read; HANDLE hFile = ::CreateFileW(SysUTF8ToWide(path).c_str(), GENERIC_READ,