Core add read and write tests

CMakeLists.txt

@@ -1677,9 +1677,9 @@ if (WIN32)
   set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
 
   if (MSVC)
-    set(SYSTEM_LIBS ${SYSTEM_LIBS} Crypt32 WS2_32 bcrypt User32 kernel32)
+    set(SYSTEM_LIBS ${SYSTEM_LIBS} Crypt32 WS2_32 bcrypt User32 shell32 kernel32)
   else()
-    set(SYSTEM_LIBS ${SYSTEM_LIBS} crypt32 ws2_32 mswsock bcrypt user32 kernel32)
+    set(SYSTEM_LIBS ${SYSTEM_LIBS} crypt32 ws2_32 mswsock bcrypt user32 shell32 kernel32)
   endif()
 
   # Don't use it even if we are compiling with gnu extensions

src/core/utils.cpp

@@ -225,6 +225,41 @@ void SetExecutablePath(const std::string& exe_path) {
   GetExecutablePath(&new_exe_path);
   absl::flags_internal::SetProgramInvocationName(new_exe_path);
 }
+
+bool GetTempDir(std::string *path) {
+  const char* tmp = getenv("TMPDIR");
+  if (tmp) {
+    *path = tmp;
+    return true;
+  }
+#if 0 // BUILDFLAG(IS_ANDROID)
+  return PathService::Get(DIR_CACHE, path);
+#else
+#if defined(__ANDROID__)
+  *path = "/data/local/tmp";
+#else
+  *path = "/tmp";
+#endif
+  return true;
+#endif
+}
+
+std::string GetHomeDir() {
+  const char* home_dir = getenv("HOME");
+  if (home_dir && home_dir[0]) {
+    return home_dir;
+  }
+#if defined(__ANDROID__)
+  DLOG(WARNING) << "OS_ANDROID: Home directory lookup not yet implemented.";
+#endif
+  // Fall back on temp dir if no home directory is defined.
+  std::string rv;
+  if (GetTempDir(&rv)) {
+    return rv;
+  }
+  // Last resort.
+  return "/tmp";
+}
 #endif
 
 /*
@@ -272,6 +307,22 @@ ssize_t ReadFileToBuffer(const std::string& path, char* buf, size_t buf_len) {
   return ret;
 }
 
+static bool WriteFileDescriptor(int fd, std::string_view data) {
+  // Allow for partial writes.
+  ssize_t bytes_written_total = 0;
+  ssize_t size = static_cast<ssize_t>(data.size());
+  DCHECK_LE(data.size(), static_cast<size_t>(SSIZE_MAX)); // checked_cast
+  for (ssize_t bytes_written_partial = 0; bytes_written_total < size;
+       bytes_written_total += bytes_written_partial) {
+    bytes_written_partial =
+        HANDLE_EINTR(::write(fd, data.data() + bytes_written_total,
+                             static_cast<size_t>(size - bytes_written_total)));
+    if (bytes_written_partial < 0)
+      return false;
+  }
+  return true;
+}
+
 ssize_t WriteFileWithBuffer(const std::string& path,
                             const char* buf,
                             size_t buf_len) {
@@ -280,7 +331,9 @@ ssize_t WriteFileWithBuffer(const std::string& path,
   if (fd < 0) {
     return false;
   }
-  ssize_t ret = HANDLE_EINTR(::write(fd, buf, buf_len));
+
+  ssize_t ret = WriteFileDescriptor(fd, std::string_view(buf, buf_len))
+    ? buf_len : -1;
 
   if (IGNORE_EINTR(close(fd)) < 0) {
     return -1;

src/core/utils.hpp

@@ -155,6 +155,16 @@ void SetExecutablePath(const std::string& exe_path);
 void SetExecutablePath(const std::wstring& exe_path);
 #endif
 
+bool GetTempDir(std::string *path);
+#ifdef _WIN32
+bool GetTempDir(std::wstring *path);
+#endif
+
+std::string GetHomeDir();
+#ifdef _WIN32
+std::wstring GetHomeDirW();
+#endif
+
 bool Net_ipv6works();
 
 #ifdef _MSC_VER

src/core/utils_fs.cpp

@@ -3,10 +3,13 @@
 #include "core/utils.hpp"
 #include "core/utils_fs.hpp"
 
-#if !defined(_WIN32_WINNT) || _WIN32_WINNT >= 0x601
 #include <filesystem>
-#else
+
+#ifdef _WIN32
 #include <windows.h>
+#else
+#include <unistd.h>
+#include <sys/stat.h>
 #endif
 
 namespace yass {
@@ -53,17 +56,6 @@ bool CreateDirectories(const std::string& path) {
   }
   return true;
 }
-
-bool RemoveFile(const std::string& path) {
-  std::error_code ec;
-  std::filesystem::path p(path);
-  std::filesystem::remove(p, ec);
-  if (ec) {
-    return false;
-  }
-  return true;
-}
-
 #else
 
 bool IsFile(const std::string& path) {
@@ -94,15 +86,55 @@ bool CreateDirectories(const std::string& path) {
   }
   return true;
 }
+#endif
 
+#ifndef _WIN32
 bool RemoveFile(const std::string& path) {
-  std::wstring wpath = SysUTF8ToWide(path);
-  if (::DeleteFileW(wpath.c_str()) == FALSE) {
-    return false;
+  const char* path_str = path.c_str();
+  struct stat file_info {};
+  if (::lstat(path_str, &file_info) != 0) {
+    // The Windows version defines this condition as success.
+    return (errno == ENOENT);
   }
-  return true;
+  if (S_ISDIR(file_info.st_mode)) {
+    return (rmdir(path_str) == 0) || (errno == ENOENT);
+  } else {
+    return (unlink(path_str) == 0) || (errno == ENOENT);
+  }
+}
+#else
+// Returns the Win32 last error code or ERROR_SUCCESS if the last error code is
+// ERROR_FILE_NOT_FOUND or ERROR_PATH_NOT_FOUND. This is useful in cases where
+// the absence of a file or path is a success condition (e.g., when attempting
+// to delete an item in the filesystem).
+static DWORD ReturnLastErrorOrSuccessOnNotFound() {
+  const DWORD error_code = ::GetLastError();
+  return (error_code == ERROR_FILE_NOT_FOUND ||
+          error_code == ERROR_PATH_NOT_FOUND)
+             ? ERROR_SUCCESS
+             : error_code;
 }
 
+bool RemoveFile(const std::string& path) {
+  std::wstring wpath = SysUTF8ToWide(path);
+  const DWORD attr = ::GetFileAttributesW(wpath.c_str());
+  if (attr == INVALID_FILE_ATTRIBUTES)
+    return ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS;
+  // Clear the read-only bit if it is set.
+  if ((attr & FILE_ATTRIBUTE_READONLY) &&
+      !::SetFileAttributesW(wpath.c_str(),
+                            attr & ~DWORD{FILE_ATTRIBUTE_READONLY})) {
+    // It's possible for |path| to be gone now under a race with other deleters.
+    return ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS;
+  }
+  if (attr & FILE_ATTRIBUTE_DIRECTORY) {
+    return ::RemoveDirectoryW(wpath.c_str()) == TRUE ? true :
+      ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS;
+  } else {
+    return ::DeleteFileW(wpath.c_str()) == TRUE ? true :
+      ReturnLastErrorOrSuccessOnNotFound() == ERROR_SUCCESS;
+  }
+}
 #endif
 
 } // yass namespace

src/core/utils_mac.mm

@@ -9,6 +9,7 @@
 
 #include <AvailabilityMacros.h>
 #include <CoreFoundation/CoreFoundation.h>
+#import <Foundation/Foundation.h>
 
 #include <errno.h>
 #include <locale.h>
@@ -22,6 +23,7 @@
 #include <sys/mman.h>  // For mlock.
 #include <sys/resource.h>
 
+#include <base/strings/sys_string_conversions.h>
 #include "core/logging.hpp"
 
 #ifndef VM_MEMORY_MALLOC_PROB_GUARD
@@ -528,4 +530,34 @@ void SetExecutablePath(const std::string& exe_path) {
   absl::flags_internal::SetProgramInvocationName(new_exe_path);
 }
 
+bool GetTempDir(std::string *path) {
+  const char* env_tmpdir = getenv("TMPDIR");
+  if (env_tmpdir) {
+    *path = env_tmpdir;
+    return true;
+  }
+  NSString* tmp = NSTemporaryDirectory();
+  if (tmp == nil) {
+    return false;
+  }
+  *path = gurl_base::SysNSStringToUTF8(tmp);
+  return true;
+}
+
+std::string GetHomeDir() {
+  NSString* tmp = NSHomeDirectory();
+  if (tmp != nil) {
+    auto path = gurl_base::SysNSStringToUTF8(tmp);
+    if (!path.empty()) {
+      return path;
+    }
+  }
+  // Fall back on temp dir if no home directory is defined.
+  std::string rv;
+  if (GetTempDir(&rv)) {
+    return rv;
+  }
+  // Last resort.
+  return "/tmp";
+}
 #endif  // __APPLE__

src/core/utils_test.cpp

@@ -3,12 +3,21 @@
 
 #include <gtest/gtest.h>
 
+#include "core/logging.hpp"
+#include "core/process_utils.hpp"
+#include "core/rand_util.hpp"
 #include "core/utils.hpp"
+#include "core/utils_fs.hpp"
 
 #ifdef _WIN32
 #include <windows.h>
 #endif
 
+#include <absl/strings/str_format.h>
+#include <filesystem>
+
+using namespace yass;
+
 TEST(UtilsTest, Dirname) {
   ASSERT_EQ(Dirname("a/b/prog/file.cc"), "a/b/prog");
   ASSERT_EQ(Dirname("a/b/prog//"), "a/b");
@@ -85,3 +94,38 @@ TEST(UtilsTest, StringToInteger) {
   i = StringToInteger(std::string(s4, 4));
   ASSERT_FALSE(i.ok()) << i.status();
 }
+
+TEST(UtilsTest, GetTempDir) {
+  std::string tmp_dir;
+  ASSERT_TRUE(GetTempDir(&tmp_dir));
+  ASSERT_FALSE(tmp_dir.empty());
+  LOG(ERROR) << "tmp_dir: " << tmp_dir;
+}
+
+TEST(UtilsTest, GetHomeDir) {
+  std::string home_dir = GetHomeDir();
+  ASSERT_FALSE(home_dir.empty());
+  LOG(ERROR) << "home_dir: " << home_dir;
+}
+
+TEST(UtilsTest, ReadFileAndWrite4K) {
+  std::string buf, buf2;
+  buf.resize(4096);
+  buf2.resize(4096);
+  RandBytes(buf.data(), buf.size());
+  int tmp_suffix;
+  RandBytes(&tmp_suffix, sizeof(tmp_suffix));
+  auto tmp_name = absl::StrFormat("read_write_file-%u-%d", GetPID(), tmp_suffix);
+  auto tmp_dir = std::filesystem::path(::testing::TempDir());
+#ifdef _WIN32
+  std::string tmp = SysWideToUTF8(tmp_dir / tmp_name);
+#else
+  std::string tmp = tmp_dir / tmp_name;
+#endif
+
+  ASSERT_TRUE(WriteFileWithBuffer(tmp, buf.c_str(), buf.size()));
+  ASSERT_TRUE(ReadFileToBuffer(tmp, buf2.data(), buf2.size()+1));
+  ASSERT_EQ(buf, buf2);
+
+  ASSERT_TRUE(RemoveFile(tmp));
+}

src/core/utils_win.cpp

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2022 Chilledheart  */
 
+#ifdef _WIN32
 
 // We use dynamic loading for below functions
 #define GetProductInfo GetProductInfoHidden
@@ -10,9 +11,11 @@
 #include "core/compiler_specific.hpp"
 #include "core/logging.hpp"
 
-#include <absl/flags/internal/program_name.h>
+#include <windows.h>
+#include <shellapi.h>
+#include <shlobj.h>
 
-#ifdef _WIN32
+#include <absl/flags/internal/program_name.h>
 
 #define MAKE_WIN_VER(major, minor, build_number) \
     (((major) << 24) | ((minor) << 16) | (build_number))
@@ -710,6 +713,55 @@ void SetExecutablePath(const std::wstring& exe_path) {
   absl::flags_internal::SetProgramInvocationName(new_exe_path);
 }
 
+bool GetTempDir(std::string *path) {
+  std::wstring wpath;
+  path->clear();
+  if (!GetTempDir(&wpath)) {
+    return false;
+  }
+  *path = SysWideToUTF8(wpath);
+  return true;
+}
+
+bool GetTempDir(std::wstring *path) {
+  wchar_t temp_path[MAX_PATH + 1];
+  DWORD path_len = ::GetTempPathW(MAX_PATH, temp_path);
+  // If the function succeeds, the return value is the length,
+  // in TCHARs, of the string copied to lpBuffer,
+  // not including the terminating null character.
+  if (path_len >= MAX_PATH || path_len <= 0)
+    return false;
+  // TODO(evanm): the old behavior of this function was to always strip the
+  // trailing slash.  We duplicate this here, but it shouldn't be necessary
+  // when everyone is using the appropriate FilePath APIs.
+  if (temp_path[path_len-1] == L'\\') {
+    temp_path[path_len-1] = L'\0';
+    --path_len;
+  }
+  *path = std::wstring(temp_path, path_len);
+  DCHECK_NE((*path)[path_len-1], L'\0');
+  return true;
+}
+
+std::string GetHomeDir() {
+  return SysWideToUTF8(GetHomeDirW());
+}
+
+std::wstring GetHomeDirW() {
+  wchar_t result[MAX_PATH];
+  if (SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_PROFILE, NULL, SHGFP_TYPE_CURRENT,
+                                 result)) && result[0]) {
+    return result;
+  }
+  // Fall back to the temporary directory on failure.
+  std::wstring rv;
+  if (GetTempDir(&rv)) {
+    return rv;
+  }
+  // Last resort.
+  return L"C:\\";
+}
+
 ssize_t ReadFileToBuffer(const std::string& path, char* buf, size_t buf_len) {
   DWORD read;
   HANDLE hFile = ::CreateFileW(SysUTF8ToWide(path).c_str(), GENERIC_READ,