Merge gvfs-helper prefetch feature

Includes these pull requests:

 #227
 #228
 #229
 #231
 #240

Signed-off-by: Derrick Stolee <dstolee@microsoft.com>

Documentation/config/core.txt

@@ -776,6 +776,10 @@ core.gvfs::
 		is first accessed and brought down to the client. Git.exe can't
 		currently tell the first access vs subsequent accesses so this
 		flag just blocks them from occurring at all.
+	GVFS_PREFETCH_DURING_FETCH::
+		Bit value 128
+		While performing a `git fetch` command, use the gvfs-helper to
+		perform a "prefetch" of commits and trees.
 --
 
 core.useGvfsHelper::

Makefile

@@ -2822,7 +2822,7 @@ gettext.sp gettext.s gettext.o: GIT-PREFIX
 gettext.sp gettext.s gettext.o: EXTRA_CPPFLAGS = \
 	-DGIT_LOCALE_PATH='"$(localedir_relative_SQ)"'
 
-http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp: SP_EXTRA_FLAGS += \
+http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp gvfs-helper.sp: SP_EXTRA_FLAGS += \
 	-DCURL_DISABLE_TYPECHECK
 
 pack-revindex.sp: SP_EXTRA_FLAGS += -Wno-memcpy-max-count

builtin/fetch.c

@@ -18,6 +18,8 @@
 #include "string-list.h"
 #include "remote.h"
 #include "transport.h"
+#include "gvfs.h"
+#include "gvfs-helper-client.h"
 #include "run-command.h"
 #include "parse-options.h"
 #include "sigchain.h"
@@ -1145,6 +1147,13 @@ static int store_updated_refs(struct display_state *display_state,
 
 		opt.exclude_hidden_refs_section = "fetch";
 		rm = ref_map;
+
+		/*
+		 * Before checking connectivity, be really sure we have the
+		 * latest pack-files loaded into memory.
+		 */
+		reprepare_packed_git(the_repository);
+
 		if (check_connected(iterate_ref_map, &rm, &opt)) {
 			rc = error(_("%s did not send all necessary objects\n"),
 				   display_state->url);
@@ -2369,6 +2378,9 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 	}
 	string_list_remove_duplicates(&list, 0);
 
+	if (core_gvfs & GVFS_PREFETCH_DURING_FETCH)
+		gh_client__prefetch(0, NULL);
+
 	if (negotiate_only) {
 		struct oidset acked_commits = OIDSET_INIT;
 		struct oidset_iter iter;

gvfs-helper-client.c

@@ -26,13 +26,14 @@ static struct hashmap gh_server__subprocess_map;
 static struct object_directory *gh_client__chosen_odb;
 
 /*
- * The "objects" capability has 2 verbs: "get" and "post".
+ * The "objects" capability has verbs: "get" and "post" and "prefetch".
  */
 #define CAP_OBJECTS      (1u<<1)
 #define CAP_OBJECTS_NAME "objects"
 
 #define CAP_OBJECTS__VERB_GET1_NAME "get"
 #define CAP_OBJECTS__VERB_POST_NAME "post"
+#define CAP_OBJECTS__VERB_PREFETCH_NAME "prefetch"
 
 static int gh_client__start_fn(struct subprocess_entry *subprocess)
 {
@@ -131,6 +132,44 @@ static int gh_client__send__objects_get(struct child_process *process,
 	return 0;
 }
 
+/*
+ * Send a request to gvfs-helper to prefetch packfiles from either the
+ * cache-server or the main Git server using "/gvfs/prefetch".
+ *
+ *     objects.prefetch LF
+ *     [<seconds-since_epoch> LF]
+ *     <flush>
+ */
+static int gh_client__send__objects_prefetch(struct child_process *process,
+					     timestamp_t seconds_since_epoch)
+{
+	int err;
+
+	/*
+	 * We assume that all of the packet_ routines call error()
+	 * so that we don't have to.
+	 */
+
+	err = packet_write_fmt_gently(
+		process->in,
+		(CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_PREFETCH_NAME "\n"));
+	if (err)
+		return err;
+
+	if (seconds_since_epoch) {
+		err = packet_write_fmt_gently(process->in, "%" PRItime "\n",
+					      seconds_since_epoch);
+		if (err)
+			return err;
+	}
+
+	err = packet_flush_gently(process->in);
+	if (err)
+		return err;
+
+	return 0;
+}
+
 /*
  * Update the loose object cache to include the newly created
  * object.
@@ -178,7 +217,7 @@ static void gh_client__update_packed_git(const char *line)
 }
 
 /*
- * Both CAP_OBJECTS verbs return the same format response:
+ * CAP_OBJECTS verbs return the same format response:
  *
  *    <odb>
  *    <data>*
@@ -218,6 +257,8 @@ static int gh_client__objects__receive_response(
 	const char *v1;
 	char *line;
 	int len;
+	int nr_loose = 0;
+	int nr_packfile = 0;
 	int err = 0;
 
 	while (1) {
@@ -236,13 +277,13 @@ static int gh_client__objects__receive_response(
 		else if (starts_with(line, "packfile")) {
 			gh_client__update_packed_git(line);
 			ghc |= GHC__CREATED__PACKFILE;
-			*p_nr_packfile += 1;
+			nr_packfile++;
 		}
 
 		else if (starts_with(line, "loose")) {
 			gh_client__update_loose_cache(line);
 			ghc |= GHC__CREATED__LOOSE;
-			*p_nr_loose += 1;
+			nr_loose++;
 		}
 
 		else if (starts_with(line, "ok"))
@@ -256,6 +297,8 @@ static int gh_client__objects__receive_response(
 	}
 
 	*p_ghc = ghc;
+	*p_nr_loose = nr_loose;
+	*p_nr_packfile = nr_packfile;
 
 	return err;
 }
@@ -312,7 +355,7 @@ static struct gh_server__process *gh_client__find_long_running_process(
 	/*
 	 * Find an existing long-running process with the above command
 	 * line -or- create a new long-running process for this and
-	 * subsequent 'get' requests.
+	 * subsequent requests.
 	 */
 	if (!gh_server__subprocess_map_initialized) {
 		gh_server__subprocess_map_initialized = 1;
@@ -349,10 +392,14 @@ static struct gh_server__process *gh_client__find_long_running_process(
 
 void gh_client__queue_oid(const struct object_id *oid)
 {
-	// TODO consider removing this trace2.  it is useful for interactive
-	// TODO debugging, but may generate way too much noise for a data
-	// TODO event.
-	trace2_printf("gh_client__queue_oid: %s", oid_to_hex(oid));
+	/*
+	 * Keep this trace as a printf only, so that it goes to the
+	 * perf log, but not the event log.  It is useful for interactive
+	 * debugging, but generates way too much (unuseful) noise for the
+	 * database.
+	 */
+	if (trace2_is_enabled())
+		trace2_printf("gh_client__queue_oid: %s", oid_to_hex(oid));
 
 	if (!oidset_insert(&gh_client__oidset_queued, oid))
 		gh_client__oidset_count++;
@@ -433,10 +480,14 @@ int gh_client__get_immediate(const struct object_id *oid,
 	int nr_packfile = 0;
 	int err = 0;
 
-	// TODO consider removing this trace2.  it is useful for interactive
-	// TODO debugging, but may generate way too much noise for a data
-	// TODO event.
-	trace2_printf("gh_client__get_immediate: %s", oid_to_hex(oid));
+	/*
+	 * Keep this trace as a printf only, so that it goes to the
+	 * perf log, but not the event log.  It is useful for interactive
+	 * debugging, but generates way too much (unuseful) noise for the
+	 * database.
+	 */
+	if (trace2_is_enabled())
+		trace2_printf("gh_client__get_immediate: %s", oid_to_hex(oid));
 
 	entry = gh_client__find_long_running_process(CAP_OBJECTS);
 	if (!entry)
@@ -465,3 +516,55 @@ int gh_client__get_immediate(const struct object_id *oid,
 
 	return err;
 }
+
+/*
+ * Ask gvfs-helper to prefetch commits-and-trees packfiles since a
+ * given timestamp.
+ *
+ * If seconds_since_epoch is zero, gvfs-helper will scan the ODB for
+ * the last received prefetch and ask for ones newer than that.
+ */
+int gh_client__prefetch(timestamp_t seconds_since_epoch,
+			int *nr_packfiles_received)
+{
+	struct gh_server__process *entry;
+	struct child_process *process;
+	enum gh_client__created ghc;
+	int nr_loose = 0;
+	int nr_packfile = 0;
+	int err = 0;
+
+	entry = gh_client__find_long_running_process(CAP_OBJECTS);
+	if (!entry)
+		return -1;
+
+	trace2_region_enter("gh-client", "objects/prefetch", the_repository);
+	trace2_data_intmax("gh-client", the_repository, "prefetch/since",
+			   seconds_since_epoch);
+
+	process = &entry->subprocess.process;
+
+	sigchain_push(SIGPIPE, SIG_IGN);
+
+	err = gh_client__send__objects_prefetch(process, seconds_since_epoch);
+	if (!err)
+		err = gh_client__objects__receive_response(
+			process, &ghc, &nr_loose, &nr_packfile);
+
+	sigchain_pop(SIGPIPE);
+
+	if (err) {
+		subprocess_stop(&gh_server__subprocess_map,
+				(struct subprocess_entry *)entry);
+		FREE_AND_NULL(entry);
+	}
+
+	trace2_data_intmax("gh-client", the_repository,
+			   "prefetch/packfile_count", nr_packfile);
+	trace2_region_leave("gh-client", "objects/prefetch", the_repository);
+
+	if (nr_packfiles_received)
+		*nr_packfiles_received = nr_packfile;
+
+	return err;
+}

gvfs-helper-client.h

@@ -66,4 +66,22 @@ void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr);
  */
 int gh_client__drain_queue(enum gh_client__created *p_ghc);
 
+/*
+ * Ask `gvfs-helper server` to fetch any "prefetch packs"
+ * available on the server more recent than the requested time.
+ *
+ * If seconds_since_epoch is zero, gvfs-helper will scan the ODB for
+ * the last received prefetch and ask for ones newer than that.
+ *
+ * A long-running background process is used to subsequent requests
+ * (either prefetch or regular immediate/queued requests) more efficient.
+ *
+ * One or more packfiles will be created in the shared-cache ODB.
+ *
+ * Returns 0 on success, -1 on error.  Optionally also returns the
+ * number of prefetch packs received.
+ */
+int gh_client__prefetch(timestamp_t seconds_since_epoch,
+			int *nr_packfiles_received);
+
 #endif /* GVFS_HELPER_CLIENT_H */