git-annex tests (#13)

Fixes #11

Tests:

* `git annex init`
* `git annex copy --from origin`
* `git annex copy --to origin`

over:

* ssh

for:

* the owner
* a collaborator
* a read-only collaborator
* a stranger

in a

* public repo
* private repo

And then confirms:

* Deletion of the remote repo (to ensure lockdown isn't messing with us: https://git-annex.branchable.com/internals/lockdown/#comment-0cc5225dc5abe8eddeb843bfd2fdc382)

------

To support all this:

* Add util.FileCmp()
* Patch withKeyFile() so it can be nested in other copies of itself

-------

Many thanks to Mathieu for giving style tips and catching several bugs,
including a subtle one in util.filecmp() which neutered it.

Co-authored-by: Mathieu Guay-Paquet <mathieu.guay-paquet@polymtl.ca>

.github/workflows/pull-db-tests.yml

@@ -46,6 +46,7 @@ jobs:
       - name: Add hosts to /etc/hosts
         run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts'
       - run: make deps-backend
+      - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
       - run: make backend
         env:
           TAGS: bindata
@@ -69,6 +70,7 @@ jobs:
           go-version-file: go.mod
           check-latest: true
       - run: make deps-backend
+      - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
       - run: make backend
         env:
           TAGS: bindata gogit sqlite sqlite_unlock_notify
@@ -172,6 +174,7 @@ jobs:
       - name: Add hosts to /etc/hosts
         run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts'
       - run: make deps-backend
+      - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
       - run: make backend
         env:
           TAGS: bindata
@@ -205,6 +208,7 @@ jobs:
       - name: Add hosts to /etc/hosts
         run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql" | sudo tee -a /etc/hosts'
       - run: make deps-backend
+      - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex
       - run: make backend
         env:
           TAGS: bindata

Makefile

@@ -8,7 +8,7 @@ self := $(location)
 	@tmpdir=`mktemp --tmpdir -d` ; \
 	echo Using temporary directory $$tmpdir for test repositories ; \
 	USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \
-	STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS
+	STATUS=$$? ; chmod -R +w "$$tmpdir" && rm -r "$$tmpdir" ; exit $$STATUS
 
 else
 

modules/util/filecmp.go

@@ -0,0 +1,87 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package util
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+// Decide if two files have the same contents or not.
+// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default.
+// *Follows* symlinks.
+//
+// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'.
+//
+// derived from https://stackoverflow.com/a/30038571
+// under CC-BY-SA-4.0 by several contributors
+func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) {
+	if chunkSize == 0 {
+		chunkSize = 4 * 1024
+	}
+
+	// shortcuts: check file metadata
+	stat1, err := os.Stat(file1)
+	if err != nil {
+		return false, err
+	}
+
+	stat2, err := os.Stat(file2)
+	if err != nil {
+		return false, err
+	}
+
+	// are inputs are literally the same file?
+	if os.SameFile(stat1, stat2) {
+		return true, nil
+	}
+
+	// do inputs at least have the same size?
+	if stat1.Size() != stat2.Size() {
+		return false, nil
+	}
+
+	// long way: compare contents
+	f1, err := os.Open(file1)
+	if err != nil {
+		return false, err
+	}
+	defer f1.Close()
+
+	f2, err := os.Open(file2)
+	if err != nil {
+		return false, err
+	}
+	defer f2.Close()
+
+	b1 := make([]byte, chunkSize)
+	b2 := make([]byte, chunkSize)
+	for {
+		n1, err1 := io.ReadFull(f1, b1)
+		n2, err2 := io.ReadFull(f2, b2)
+
+		// https://pkg.go.dev/io#Reader
+		// > Callers should always process the n > 0 bytes returned
+		// > before considering the error err. Doing so correctly
+		// > handles I/O errors that happen after reading some bytes
+		// > and also both of the allowed EOF behaviors.
+
+		if !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, nil
+		}
+
+		if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) {
+			return true, nil
+		}
+
+		// some other error, like a dropped network connection or a bad transfer
+		if err1 != nil {
+			return false, err1
+		}
+		if err2 != nil {
+			return false, err2
+		}
+	}
+}

tests/integration/api_helper_for_declarative_test.go

@@ -21,6 +21,7 @@ import (
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/services/forms"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -462,3 +463,27 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r
 		ctx.Session.MakeRequest(t, req, http.StatusNoContent)
 	}
 }
+
+// generate and activate an ssh key for the user attached to the APITestContext
+// TODO: pick a better name; golang doesn't do method overriding.
+func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) {
+	// we need to have write:public_key to do this step
+	// the easiest way is to create a throwaway ctx that is identical but only has that permission
+	ctxKeyWriter := ctx
+	ctxKeyWriter.Token = getTokenForLoggedInUser(t, ctx.Session, auth.AccessTokenScopeWriteUser)
+
+	keyName := "One of " + ctx.Username + "'s keys: #" + uuid.New().String()
+	withKeyFile(t, keyName, func(keyFile string) {
+		var key api.PublicKey
+
+		doAPICreateUserKey(ctxKeyWriter, keyName, keyFile,
+			func(t *testing.T, _key api.PublicKey) {
+				// save the key ID so we can delete it at the end
+				key = _key
+			})(t)
+
+		defer doAPIDeleteUserKey(ctxKeyWriter, key.ID)(t)
+
+		callback()
+	})
+}