test: build with large file transfer

Dockerfile

@@ -4,8 +4,8 @@ ARG GO_VERSION=1.22
 ARG ALPINE_VERSION=3.20
 ARG XX_VERSION=1.4.0
 
-ARG BUILDKIT_VERSION=v0.16.0
-ARG BUILDX_VERSION=v0.17.1
+ARG BUILDKIT_VERSION=v0.17.1
+ARG BUILDX_VERSION=v0.18.0
 ARG REGISTRY_VERSION=v2.8.3
 
 # named contexts

cmd/refcandidates/main.go

@@ -3,11 +3,9 @@ package main
 import (
 	"encoding/json"
 	"log"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/alecthomas/kong"
 	"github.com/moby/buildkit-bench/util/candidates"
@@ -105,22 +103,22 @@ func setGhaOutput(name string, c *candidates.Candidates) error {
 		})
 	}
 
-	if gha.IsPullRequestEvent() {
-		r := rand.New(rand.NewSource(time.Now().UnixNano()))
-		if len(includes) > 2 {
-			s := make([]include, 0, 2)
-			si := make(map[int]struct{})
-			for len(s) < 2 {
-				idx := r.Intn(len(includes))
-				if _, exists := si[idx]; !exists {
-					si[idx] = struct{}{}
-					s = append(s, includes[idx])
-				}
-			}
-			includes = s
-		}
-		log.Printf("Reducing candidates to %d for pull request event in GHA output", len(includes))
-	}
+	//if gha.IsPullRequestEvent() {
+	//	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	//	if len(includes) > 2 {
+	//		s := make([]include, 0, 2)
+	//		si := make(map[int]struct{})
+	//		for len(s) < 2 {
+	//			idx := r.Intn(len(includes))
+	//			if _, exists := si[idx]; !exists {
+	//				si[idx] = struct{}{}
+	//				s = append(s, includes[idx])
+	//			}
+	//		}
+	//		includes = s
+	//	}
+	//	log.Printf("Reducing candidates to %d for pull request event in GHA output", len(includes))
+	//}
 
 	dt, err := json.Marshal(includes)
 	if err != nil {

test/build_test.go

@@ -2,6 +2,7 @@ package test
 
 import (
 	"fmt"
+	"math/rand"
 	"strings"
 	"sync"
 	"testing"
@@ -55,6 +56,7 @@ func BenchmarkBuild(b *testing.B) {
 		benchmarkBuildHighParallelization128x,
 		benchmarkBuildFileTransfer,
 		benchmarkBuildFileTransferCachedWithIteration,
+		benchmarkBuildLargeFileTransfer,
 		benchmarkBuildEmulator,
 		benchmarkBuildExportUncompressed,
 		benchmarkBuildExportGzip,
@@ -215,6 +217,37 @@ RUN du -sh . && tree .
 	})
 }
 
+func benchmarkBuildLargeFileTransfer(b *testing.B, sb testutil.Sandbox) {
+	dockerfile := []byte(`
+FROM busybox:latest
+WORKDIR /src
+COPY . .
+RUN du -sh .
+`)
+
+	var contextDirLargeAppliers []fstest.Applier
+	contextDirLargeAppliers = append(contextDirLargeAppliers,
+		fstest.CreateDir("subdir9", 0755),
+		fstest.CreateFile("subdir9/largefile1.txt", make([]byte, 1024*1024*200), 0600), // 200MB file
+		fstest.CreateFile("subdir9/largefile2.txt", make([]byte, 1024*1024*400), 0600), // 400MB file
+	)
+	for i := 0; i < 10000; i++ {
+		fsize := rand.Intn(100*1024-5*1024) + 5*1024 // random size between 5KB and 100KB
+		contextDirLargeAppliers = append(contextDirLargeAppliers, fstest.CreateFile(fmt.Sprintf("subdir9/file%d.txt", i+5), make([]byte, fsize), 0600))
+	}
+	dir := tmpdir(b,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		fstest.Apply(append(contextDirLargeAppliers, contextDirApplier)...),
+	)
+	reportBuildkitdAlloc(b, sb, func() {
+		b.StartTimer()
+		out, err := buildxBuildCmd(sb, withArgs(dir))
+		b.StopTimer()
+		sb.WriteLogFile(b, "buildx", []byte(out))
+		require.NoError(b, err, out)
+	})
+}
+
 func benchmarkBuildFileTransferCachedWithIteration(b *testing.B, sb testutil.Sandbox) {
 	dockerfile := []byte(`
 FROM busybox:latest

testconfig.yml

@@ -104,6 +104,16 @@ runs:
         alloc:
           description: Allocated memory (bytes)
           chart: boxplot
+    BenchmarkBuildLargeFileTransfer:
+      description: Build with large file transfer
+      count: 4
+      metrics:
+        duration:
+          description: Time (s)
+          chart: boxplot
+        alloc:
+          description: Allocated memory (bytes)
+          chart: boxplot
     BenchmarkBuildEmulator:
       description: Build with emulator
       count: 10