explicitly FlushViewOfFile() on windows when closing a file and perio…

…dically the file with most dirty bytes

ChangeLog

@@ -1,3 +1,4 @@
+	* on windows, explicitly flush memory mapped files periodically
 	* fix build with WolfSSL
 	* fix issue where incoming uTP connections were not accepted over SOCKS5
 	* fix several issues in handling of checking files of v2 torrents, esp. from magnet links

Jamfile

@@ -309,7 +309,7 @@ rule building ( properties * )
 	if <toolset>msvc in $(properties) || <toolset>intel-win in $(properties)
 	{
 		# allow larger .obj files (with more sections)
-		result += <cflags>/bigobj ;
+		result += <cxxflags>/bigobj ;
 	}
 
 	if <toolset>gcc in $(properties) && <target-os>windows in $(properties)

include/libtorrent/aux_/file_view_pool.hpp

@@ -110,6 +110,12 @@ namespace aux {
 
 		void close_oldest();
 
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+		void flush_next_file();
+		void record_file_write(storage_index_t st, file_index_t file_index
+			, uint64_t pages);
+#endif
+
 	private:
 
 		std::shared_ptr<file_mapping> remove_oldest(std::unique_lock<std::mutex>&);
@@ -140,6 +146,9 @@ namespace aux {
 			file_id key;
 			std::shared_ptr<file_mapping> mapping;
 			time_point last_use{aux::time_now()};
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+			std::uint64_t dirty_bytes;
+#endif
 			open_mode_t mode{};
 		};
 
@@ -150,6 +159,10 @@ namespace aux {
 			mi::ordered_unique<mi::member<file_entry, file_id, &file_entry::key>>,
 			// look up files by least recently used
 			mi::sequenced<>
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+			// look up files with dirty pages
+			, mi::ordered_non_unique<mi::member<file_entry, std::uint64_t, &file_entry::dirty_bytes>>
+#endif
 			>
 		>;
 

include/libtorrent/aux_/mmap.hpp

@@ -122,6 +122,10 @@ namespace aux {
 #endif
 			);
 
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+		void flush();
+#endif
+
 		// non-copyable
 		file_mapping(file_mapping const&) = delete;
 		file_mapping& operator=(file_mapping const&) = delete;

src/file_view_pool.cpp

@@ -269,6 +269,36 @@ namespace libtorrent { namespace aux {
 		std::unique_lock<std::mutex> l(m_mutex);
 		deferred_destruction = remove_oldest(l);
 	}
+
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+	void file_view_pool::flush_next_file()
+	{
+		std::shared_ptr<file_mapping> mapping;
+		{
+			std::unique_lock<std::mutex> l(m_mutex);
+			auto& flush_view = m_files.get<2>();
+			if (flush_view.size() == 0) return;
+
+			auto it = std::prev(flush_view.end());
+			if (it->dirty_bytes == 0) return;
+			mapping = it->mapping;
+			flush_view.modify(it, [](file_entry& e) { e.dirty_bytes = 0; });
+		}
+
+		// we invoke flush after we release the mutex
+		mapping->flush();
+	}
+
+	void file_view_pool::record_file_write(storage_index_t const st
+		, file_index_t const file_index, uint64_t const bytes)
+	{
+		std::unique_lock<std::mutex> l(m_mutex);
+		auto& key_view = m_files.get<0>();
+		auto i = key_view.find(file_id{st, file_index});
+		if (i == key_view.end()) return;
+		key_view.modify(i, [bytes](file_entry& e) { e.dirty_bytes += bytes; });
+	}
+#endif
 }
 }
 

src/mmap.cpp

@@ -638,9 +638,18 @@ file_mapping::file_mapping(file_handle file, open_mode_t const mode
 		throw_ex<storage_error>(error_code(GetLastError(), system_category()), operation_t::file_mmap);
 }
 
+void file_mapping::flush()
+{
+	if (m_mapping == nullptr) return;
+
+	// ignore errors, this is best-effort
+	FlushViewOfFile(m_mapping, static_cast<std::size_t>(m_size));
+}
+
 void file_mapping::close()
 {
 	if (m_mapping == nullptr) return;
+	flush();
 	std::lock_guard<std::mutex> l(*m_open_unmap_lock);
 	UnmapViewOfFile(m_mapping);
 	m_mapping = nullptr;

src/mmap_disk_io.cpp

@@ -314,10 +314,6 @@ struct TORRENT_EXTRA_EXPORT mmap_disk_io final
 
 	settings_interface const& m_settings;
 
-	// we call close_oldest_file on the file_pool regularly. This is the next
-	// time we should call it
-	time_point m_next_close_oldest_file = min_time();
-
 	// LRU cache of open files
 	aux::file_view_pool m_file_pool;
 
@@ -1510,6 +1506,14 @@ TORRENT_EXPORT std::unique_ptr<disk_interface> mmap_disk_io_constructor(
 		++m_num_running_threads;
 		m_stats_counters.inc_stats_counter(counters::num_running_threads, 1);
 
+		// we call close_oldest_file on the file_pool regularly. This is the next
+		// time we should call it
+		time_point next_close_oldest_file = min_time();
+
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+		time_point next_flush_file = min_time();
+#endif
+
 		for (;;)
 		{
 			aux::disk_io_job* j = nullptr;
@@ -1538,20 +1542,30 @@ TORRENT_EXPORT std::unique_ptr<disk_interface> mmap_disk_io_constructor(
 					}
 				}
 
-				if (now > m_next_close_oldest_file)
+				if (now > next_close_oldest_file)
 				{
 					seconds const interval(m_settings.get_int(settings_pack::close_file_interval));
 					if (interval <= seconds(0))
 					{
 						// check again in one minute, in case the setting changed
-						m_next_close_oldest_file = now + minutes(1);
+						next_close_oldest_file = now + minutes(1);
 					}
 					else
 					{
-						m_next_close_oldest_file = now + interval;
+						next_close_oldest_file = now + interval;
 						m_file_pool.close_oldest();
 					}
 				}
+
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+				if (now > next_flush_file)
+				{
+					// on windows we need to explicitly ask the operating system to flush
+					// dirty pages from time to time
+					m_file_pool.flush_next_file();
+					next_flush_file = now + seconds(30);
+				}
+#endif
 			}
 
 			execute_job(j);

src/mmap_storage.cpp

@@ -663,6 +663,10 @@ namespace libtorrent {
 				ret += static_cast<int>(buf.size());
 			}
 
+#if TORRENT_HAVE_MAP_VIEW_OF_FILE
+			m_pool.record_file_write(storage_index(), file_index, ret);
+#endif
+
 			// set this unconditionally in case the upper layer would like to treat
 			// short reads as errors
 			ec.operation = operation_t::file_write;