diff --git a/ChangeLog b/ChangeLog index ec5ba1c1213..0478d458dfe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * on windows, explicitly flush memory mapped files periodically * fix build with WolfSSL * fix issue where incoming uTP connections were not accepted over SOCKS5 * fix several issues in handling of checking files of v2 torrents, esp. from magnet links diff --git a/Jamfile b/Jamfile index c92e27cf9c4..315da24d448 100644 --- a/Jamfile +++ b/Jamfile @@ -309,7 +309,7 @@ rule building ( properties * ) if msvc in $(properties) || intel-win in $(properties) { # allow larger .obj files (with more sections) - result += /bigobj ; + result += /bigobj ; } if gcc in $(properties) && windows in $(properties) diff --git a/include/libtorrent/aux_/file_view_pool.hpp b/include/libtorrent/aux_/file_view_pool.hpp index a6ef66d14d0..2bf722bac23 100644 --- a/include/libtorrent/aux_/file_view_pool.hpp +++ b/include/libtorrent/aux_/file_view_pool.hpp @@ -110,6 +110,12 @@ namespace aux { void close_oldest(); +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + void flush_next_file(); + void record_file_write(storage_index_t st, file_index_t file_index + , uint64_t pages); +#endif + private: std::shared_ptr remove_oldest(std::unique_lock&); @@ -140,6 +146,9 @@ namespace aux { file_id key; std::shared_ptr mapping; time_point last_use{aux::time_now()}; +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + std::uint64_t dirty_bytes; +#endif open_mode_t mode{}; }; @@ -150,6 +159,10 @@ namespace aux { mi::ordered_unique>, // look up files by least recently used mi::sequenced<> +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + // look up files with dirty pages + , mi::ordered_non_unique> +#endif > >; diff --git a/include/libtorrent/aux_/mmap.hpp b/include/libtorrent/aux_/mmap.hpp index 254e122f3d1..4d18421c76e 100644 --- a/include/libtorrent/aux_/mmap.hpp +++ b/include/libtorrent/aux_/mmap.hpp @@ -122,6 +122,10 @@ namespace aux { #endif ); +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + void flush(); +#endif + // non-copyable file_mapping(file_mapping const&) = delete; file_mapping& operator=(file_mapping const&) = delete; diff --git a/src/file_view_pool.cpp b/src/file_view_pool.cpp index 87fc467a83a..ee59d2497c3 100644 --- a/src/file_view_pool.cpp +++ b/src/file_view_pool.cpp @@ -269,6 +269,36 @@ namespace libtorrent { namespace aux { std::unique_lock l(m_mutex); deferred_destruction = remove_oldest(l); } + +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + void file_view_pool::flush_next_file() + { + std::shared_ptr mapping; + { + std::unique_lock l(m_mutex); + auto& flush_view = m_files.get<2>(); + if (flush_view.size() == 0) return; + + auto it = std::prev(flush_view.end()); + if (it->dirty_bytes == 0) return; + mapping = it->mapping; + flush_view.modify(it, [](file_entry& e) { e.dirty_bytes = 0; }); + } + + // we invoke flush after we release the mutex + mapping->flush(); + } + + void file_view_pool::record_file_write(storage_index_t const st + , file_index_t const file_index, uint64_t const bytes) + { + std::unique_lock l(m_mutex); + auto& key_view = m_files.get<0>(); + auto i = key_view.find(file_id{st, file_index}); + if (i == key_view.end()) return; + key_view.modify(i, [bytes](file_entry& e) { e.dirty_bytes += bytes; }); + } +#endif } } diff --git a/src/mmap.cpp b/src/mmap.cpp index 836cbb9a1c6..4f823c6fcc5 100644 --- a/src/mmap.cpp +++ b/src/mmap.cpp @@ -638,9 +638,18 @@ file_mapping::file_mapping(file_handle file, open_mode_t const mode throw_ex(error_code(GetLastError(), system_category()), operation_t::file_mmap); } +void file_mapping::flush() +{ + if (m_mapping == nullptr) return; + + // ignore errors, this is best-effort + FlushViewOfFile(m_mapping, static_cast(m_size)); +} + void file_mapping::close() { if (m_mapping == nullptr) return; + flush(); std::lock_guard l(*m_open_unmap_lock); UnmapViewOfFile(m_mapping); m_mapping = nullptr; diff --git a/src/mmap_disk_io.cpp b/src/mmap_disk_io.cpp index 8899146f9f3..e495b2ca779 100644 --- a/src/mmap_disk_io.cpp +++ b/src/mmap_disk_io.cpp @@ -314,10 +314,6 @@ struct TORRENT_EXTRA_EXPORT mmap_disk_io final settings_interface const& m_settings; - // we call close_oldest_file on the file_pool regularly. This is the next - // time we should call it - time_point m_next_close_oldest_file = min_time(); - // LRU cache of open files aux::file_view_pool m_file_pool; @@ -1510,6 +1506,14 @@ TORRENT_EXPORT std::unique_ptr mmap_disk_io_constructor( ++m_num_running_threads; m_stats_counters.inc_stats_counter(counters::num_running_threads, 1); + // we call close_oldest_file on the file_pool regularly. This is the next + // time we should call it + time_point next_close_oldest_file = min_time(); + +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + time_point next_flush_file = min_time(); +#endif + for (;;) { aux::disk_io_job* j = nullptr; @@ -1538,20 +1542,30 @@ TORRENT_EXPORT std::unique_ptr mmap_disk_io_constructor( } } - if (now > m_next_close_oldest_file) + if (now > next_close_oldest_file) { seconds const interval(m_settings.get_int(settings_pack::close_file_interval)); if (interval <= seconds(0)) { // check again in one minute, in case the setting changed - m_next_close_oldest_file = now + minutes(1); + next_close_oldest_file = now + minutes(1); } else { - m_next_close_oldest_file = now + interval; + next_close_oldest_file = now + interval; m_file_pool.close_oldest(); } } + +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + if (now > next_flush_file) + { + // on windows we need to explicitly ask the operating system to flush + // dirty pages from time to time + m_file_pool.flush_next_file(); + next_flush_file = now + seconds(30); + } +#endif } execute_job(j); diff --git a/src/mmap_storage.cpp b/src/mmap_storage.cpp index 27ac8bc3f42..b5d86147bfb 100644 --- a/src/mmap_storage.cpp +++ b/src/mmap_storage.cpp @@ -663,6 +663,10 @@ namespace libtorrent { ret += static_cast(buf.size()); } +#if TORRENT_HAVE_MAP_VIEW_OF_FILE + m_pool.record_file_write(storage_index(), file_index, ret); +#endif + // set this unconditionally in case the upper layer would like to treat // short reads as errors ec.operation = operation_t::file_write;