ctlog: test recover behavior with errors on overwrite

Test recovery behavior when using conditional requests to upload immutable
files (using "If-Match" on Tigris).

A sequencing failure after uploading a tile but before updating the lock can
cause the log to get stuck:
- Sequencing starts.
- The sequencer writer tile file "A".
- The sequences fails to update the lock.
- Sequencing restarts.
- The sequencer tries to write tile file "A".
- The sequencer fails because tile file "A" already exists.

In the log this looks as follows:

    testlog_test.go:79: time=2024-03-16T17:44:35.325-07:00 level=DEBUG source=ctlog.go:751 msg="uploading partial data tile" tree_size=1 tile="{H:8 L:-1 N:0 W:1}" size=25
    testlog_test.go:79: time=2024-03-16T17:44:35.325-07:00 level=DEBUG source=ctlog.go:771 msg="uploading tree tile" old_tree_size=0 tree_size=1 tile="{H:8 L:0 N:0 W:1}" size=32
    testlog_test.go:79: time=2024-03-16T17:44:35.325-07:00 level=DEBUG source=ctlog.go:795 msg="uploading checkpoint" size=209
    testlog_test.go:79: time=2024-03-16T17:44:35.325-07:00 level=ERROR source=ctlog.go:671 msg="pool sequencing failed" old_tree_size=0 entries=1 err="fatal sequencing error\nfailing commit checkpoint for test"
    ...
    testlog_test.go:79: time=2024-03-16T17:44:35.326-07:00 level=DEBUG source=ctlog.go:751 msg="uploading partial data tile" tree_size=1 tile="{H:8 L:-1 N:0 W:1}" size=32
    testlog_test.go:79: time=2024-03-16T17:44:35.326-07:00 level=DEBUG source=ctlog.go:771 msg="uploading tree tile" old_tree_size=0 tree_size=1 tile="{H:8 L:0 N:0 W:1}" size=32
    testlog_test.go:79: time=2024-03-16T17:44:35.326-07:00 level=ERROR source=ctlog.go:671 msg="pool sequencing failed" old_tree_size=0 entries=1 err="couldn't upload a tile: key \"tile/8/0/000.p/1\" already exists"

internal/ctlog/ctlog.go

@@ -793,6 +793,11 @@ func (l *Log) sequencePool(ctx context.Context, p *pool) (err error) {
 		return fmtErrorf("couldn't sign checkpoint: %w", err)
 	}
 	l.c.Log.DebugContext(ctx, "uploading checkpoint", "size", len(checkpoint))
+
+	if testingOnlyFailCommit {
+		return errors.Join(errFatal, errors.New("failing commit checkpoint for test"))
+	}
+
 	newLock, err := l.c.Lock.Replace(ctx, l.lockCheckpoint, checkpoint)
 	if err != nil {
 		// This is a critical error, since we don't know the state of the
@@ -844,6 +849,8 @@ func (l *Log) sequencePool(ctx context.Context, p *pool) (err error) {
 
 var testingOnlyPauseSequencing func()
 
+var testingOnlyFailCommit bool
+
 // signTreeHead signs the tree and returns a checkpoint according to
 // c2sp.org/checkpoint.
 func signTreeHead(name string, logID [sha256.Size]byte, privKey *ecdsa.PrivateKey, tree treeWithTimestamp) (checkpoint []byte, err error) {

internal/ctlog/ctlog_test.go

@@ -260,6 +260,33 @@ func testReloadLog(t *testing.T, add func(*testing.T, *TestLog) func(context.Con
 	}
 }
 
+func TestRecoverLogFailCommit(t *testing.T) {
+	tl := NewEmptyTestLog(t)
+	n := int64(tileWidth + 2)
+	if testing.Short() {
+		n = 3
+	} else {
+		tl.Quiet()
+	}
+
+	// set to false to make test pass
+	tl.Config.Backend.(*MemoryBackend).errorOnOverwrite = true
+
+	for i := int64(0); i < n; i++ {
+		addCertificateFast(t, tl) // won't complain on error
+		ctlog.SetFailCommit(true)
+		if err := tl.Log.Sequence(); err == nil {
+			t.Fatal("expected sequencer failure")
+		}
+		ctlog.SetFailCommit(false)
+
+		tl = ReloadLog(t, tl)
+		addCertificate(t, tl) // will complain on error
+		fatalIfErr(t, tl.Log.Sequence())
+		tl.CheckLog()
+	}
+}
+
 func TestSubmit(t *testing.T) {
 	t.Run("Certificates", func(t *testing.T) {
 		testSubmit(t, false)

internal/ctlog/export_test.go

@@ -26,3 +26,7 @@ func PauseSequencer() {
 func ResumeSequencer() {
 	close(seqRunning)
 }
+
+func SetFailCommit(b bool) {
+	testingOnlyFailCommit = b
+}

internal/ctlog/testlog_test.go

@@ -322,7 +322,8 @@ type MemoryBackend struct {
 	mu sync.Mutex
 	m  map[string][]byte
 
-	uploads uint64
+	uploads          uint64
+	errorOnOverwrite bool
 }
 
 func NewMemoryBackend(t testing.TB) *MemoryBackend {
@@ -342,6 +343,11 @@ func (b *MemoryBackend) Upload(ctx context.Context, key string, data []byte, opt
 	}
 	b.mu.Lock()
 	defer b.mu.Unlock()
+	if opts.Immutable && b.errorOnOverwrite {
+		if _, ok := b.m[key]; ok {
+			return fmt.Errorf("key %q already exists", key)
+		}
+	}
 	b.m[key] = data
 	return nil
 }