diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 923c51a59..cadd88044 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -138,6 +138,8 @@ jobs: needs: build steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2 + - name: Create git branch + run: git switch -c harness-test-branch - name: Extract version of Go to use run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV - uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v3.1.0 diff --git a/tests/harness_test.go b/tests/harness_test.go new file mode 100644 index 000000000..af42495a9 --- /dev/null +++ b/tests/harness_test.go @@ -0,0 +1,211 @@ +// Copyright 2022 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e +// +build e2e + +package e2e + +import ( + "bytes" + "crypto/ecdsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/in-toto/in-toto-golang/in_toto" + slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" + "github.com/secure-systems-lab/go-securesystemslib/dsse" + "github.com/sigstore/rekor/pkg/generated/models" + "github.com/sigstore/rekor/pkg/types" +) + +// Make sure we can add an entry +func TestHarnessAddEntry(t *testing.T) { + // Create a random artifact and sign it. + artifactPath := filepath.Join(t.TempDir(), "artifact") + sigPath := filepath.Join(t.TempDir(), "signature.asc") + + createdX509SignedArtifact(t, artifactPath, sigPath) + dataBytes, _ := ioutil.ReadFile(artifactPath) + h := sha256.Sum256(dataBytes) + dataSHA := hex.EncodeToString(h[:]) + + // Write the public key to a file + pubPath := filepath.Join(t.TempDir(), "pubKey.asc") + if err := ioutil.WriteFile(pubPath, []byte(rsaCert), 0644); err != nil { + t.Fatal(err) + } + + // Verify should fail initially + runCliErr(t, "verify", "--type=hashedrekord", "--pki-format=x509", "--artifact-hash", dataSHA, "--signature", sigPath, "--public-key", pubPath) + + // It should upload successfully. + out := runCli(t, "upload", "--type=hashedrekord", "--pki-format=x509", "--artifact-hash", dataSHA, "--signature", sigPath, "--public-key", pubPath) + outputContains(t, out, "Created entry at") + + // Now we should be able to verify it. + out = runCli(t, "verify", "--type=hashedrekord", "--pki-format=x509", "--artifact-hash", dataSHA, "--signature", sigPath, "--public-key", pubPath) + outputContains(t, out, "Inclusion Proof:") +} + +// Make sure we can add an intoto entry +func TestHarnessAddIntoto(t *testing.T) { + td := t.TempDir() + attestationPath := filepath.Join(td, "attestation.json") + pubKeyPath := filepath.Join(td, "pub.pem") + + // Get some random data so it's unique each run + d := randomData(t, 10) + id := base64.StdEncoding.EncodeToString(d) + + it := in_toto.ProvenanceStatement{ + StatementHeader: in_toto.StatementHeader{ + Type: in_toto.StatementInTotoV01, + PredicateType: slsa.PredicateSLSAProvenance, + Subject: []in_toto.Subject{ + { + Name: "foobar", + Digest: slsa.DigestSet{ + "foo": "bar", + }, + }, + }, + }, + Predicate: slsa.ProvenancePredicate{ + Builder: slsa.ProvenanceBuilder{ + ID: "foo" + id, + }, + }, + } + + b, err := json.Marshal(it) + if err != nil { + t.Fatal(err) + } + + pb, _ := pem.Decode([]byte(ecdsaPriv)) + priv, err := x509.ParsePKCS8PrivateKey(pb.Bytes) + if err != nil { + t.Fatal(err) + } + signer, err := dsse.NewEnvelopeSigner(&IntotoSigner{ + priv: priv.(*ecdsa.PrivateKey), + }) + if err != nil { + t.Fatal(err) + } + + env, err := signer.SignPayload("application/vnd.in-toto+json", b) + if err != nil { + t.Fatal(err) + } + + eb, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + + write(t, string(eb), attestationPath) + write(t, ecdsaPub, pubKeyPath) + + // If we do it twice, it should already exist + out := runCli(t, "upload", "--artifact", attestationPath, "--type", "intoto", "--public-key", pubKeyPath) + outputContains(t, out, "Created entry at") + uuid := getUUIDFromUploadOutput(t, out) + + out = runCli(t, "get", "--uuid", uuid, "--format=json") + g := getOut{} + if err := json.Unmarshal([]byte(out), &g); err != nil { + t.Fatal(err) + } + // The attestation should be stored at /var/run/attestations/sha256:digest + + got := in_toto.ProvenanceStatement{} + if err := json.Unmarshal([]byte(g.Attestation), &got); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(it, got); diff != "" { + t.Errorf("diff: %s", diff) + } + + attHash := sha256.Sum256(b) + + intotoModel := &models.IntotoV001Schema{} + if err := types.DecodeEntry(g.Body.(map[string]interface{})["IntotoObj"], intotoModel); err != nil { + t.Errorf("could not convert body into intoto type: %v", err) + } + if intotoModel.Content == nil || intotoModel.Content.PayloadHash == nil { + t.Errorf("could not find hash over attestation %v", intotoModel) + } + recordedPayloadHash, err := hex.DecodeString(*intotoModel.Content.PayloadHash.Value) + if err != nil { + t.Errorf("error converting attestation hash to []byte: %v", err) + } + + if !bytes.Equal(attHash[:], recordedPayloadHash) { + t.Fatal(fmt.Errorf("attestation hash %v doesnt match the payload we sent %v", hex.EncodeToString(attHash[:]), + *intotoModel.Content.PayloadHash.Value)) + } + + out = runCli(t, "upload", "--artifact", attestationPath, "--type", "intoto", "--public-key", pubKeyPath) + outputContains(t, out, "Entry already exists") +} + +// Make sure we can get and verify all entries +// For attestations, make sure we can see the attestation +func TestHarnessGetAllEntriesLogIndex(t *testing.T) { + treeSize := activeTreeSize(t) + if treeSize == 0 { + t.Fatal("There are 0 entries in the log, there should be at least 2") + } + for i := 0; i < treeSize; i++ { + out := runCli(t, "get", "--log-index", fmt.Sprintf("%d", i), "--format", "json") + if !strings.Contains(out, "IntotoObj") { + continue + } + var intotoObj struct { + Attestation string + } + if err := json.Unmarshal([]byte(out), &intotoObj); err != nil { + t.Fatal(err) + } + if intotoObj.Attestation == "" { + t.Log(out) + t.Fatalf("intotoObj attestation is empty for log index %d", i) + } + t.Log("Got IntotoObj type with attestation") + } +} + +func activeTreeSize(t *testing.T) int { + out := runCliStdout(t, "loginfo", "--format", "json", "--store_tree_state", "false") + t.Log(string(out)) + var s struct { + ActiveTreeSize int + } + if err := json.Unmarshal([]byte(out), &s); err != nil { + t.Fatal(err) + } + return s.ActiveTreeSize +} diff --git a/tests/rekor-harness.sh b/tests/rekor-harness.sh index 13c4c5f80..4aa201fa7 100755 --- a/tests/rekor-harness.sh +++ b/tests/rekor-harness.sh @@ -15,6 +15,8 @@ # limitations under the License. set -e +TREE_ID="" + function start_server () { server_version=$1 current_branch=$(git rev-parse --abbrev-ref HEAD) @@ -22,12 +24,20 @@ function start_server () { if [ $(docker-compose ps | grep -c "(healthy)") == 0 ]; then echo "starting services with version $server_version" docker-compose up -d --build + sleep 30 + make rekor-cli + export TREE_ID=$(./rekor-cli loginfo --format json --rekor_server http://localhost:3000 --store_tree_state=false | jq -r .TreeID) else echo "turning down rekor and restarting at version $server_version" docker stop $(docker ps --filter name=rekor-server --format {{.ID}}) + + # Replace log in docker-compose.yml with the Tree ID we want + search="# Uncomment this for production logging" + replace="\"--trillian_log_server.tlog_id=$TREE_ID\"," + sed -i "s/$search/$replace/" docker-compose.yml + docker-compose up -d --build rekor-server fi - git checkout $current_branch count=0 echo -n "waiting up to 60 sec for system to start" @@ -35,6 +45,8 @@ function start_server () { do if [ $count -eq 6 ]; then echo "! timeout reached" + cat docker-compose.yml + docker-compose logs --no-color > /tmp/docker-compose.log exit 1 else echo -n "." @@ -42,6 +54,8 @@ function start_server () { let 'count+=1' fi done + git checkout $server_version . + git checkout $current_branch echo } @@ -51,6 +65,7 @@ function build_cli () { current_branch=$(git rev-parse --abbrev-ref HEAD) git checkout $cli_version make rekor-cli + git checkout $cli_version . git checkout $current_branch } @@ -59,30 +74,25 @@ function run_tests () { touch $REKORTMPDIR.rekor.yaml trap "rm -rf $REKORTMPDIR" EXIT - go clean -testcache - for test in $HARNESS_TESTS - do - if ! REKORTMPDIR=$REKORTMPDIR go test -run $test -v -tags=e2e ./tests/ > $REKORTMPDIR/logs ; then - cat $REKORTMPDIR/logs - docker-compose logs --no-color > /tmp/docker-compose.log - exit 1 - fi - if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then - # if we're here, we found a panic - echo "Failing due to panics detected in logs" - docker-compose logs --no-color > /tmp/docker-compose.log - exit 1 - fi - done + if ! REKORTMPDIR=$REKORTMPDIR go test -run TestHarness -v -tags=e2e ./tests/ ; then + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 + fi + if docker-compose logs --no-color | grep -q "panic: runtime error:" ; then + # if we're here, we found a panic + echo "Failing due to panics detected in logs" + docker-compose logs --no-color > /tmp/docker-compose.log + exit 1 + fi } # Get last 3 server versions git fetch origin -VERSIONS=$(git tag --sort=-version:refname | head -n 3 | tac) +NUM_VERSIONS_TO_TEST=3 +VERSIONS=$(git tag --sort=-version:refname | head -n $NUM_VERSIONS_TO_TEST | tac) echo $VERSIONS -HARNESS_TESTS="TestUploadVerify TestLogInfo TestGetCLI TestSSH TestJAR TestAPK TestIntoto TestX509 TestEntryUpload" for server_version in $VERSIONS do @@ -100,4 +110,13 @@ do done done +# Since we add two entries to the log for every test, once all tests are run we should have 2*($NUM_VERSIONS_TO_TEST^2) entries +make rekor-cli +actual=$(./rekor-cli loginfo --rekor_server http://localhost:3000 --format json --store_tree_state=false | jq -r .ActiveTreeSize) +expected=$((2*$NUM_VERSIONS_TO_TEST*$NUM_VERSIONS_TO_TEST)) +if [[ ! "$expected" == "$actual" ]]; then + echo "ERROR: Log had $actual entries instead of expected $expected" + exit 1 +fi + echo "Harness testing successful :)" diff --git a/tests/util.go b/tests/util.go index 7d78642c2..aec3b756d 100644 --- a/tests/util.go +++ b/tests/util.go @@ -74,6 +74,23 @@ func runCli(t *testing.T, arg ...string) string { return run(t, "", cli, arg...) } +func runCliStdout(t *testing.T, arg ...string) string { + t.Helper() + arg = append(arg, rekorServerFlag()) + c := exec.Command(cli, arg...) + + if os.Getenv("REKORTMPDIR") != "" { + // ensure that we use a clean state.json file for each run + c.Env = append(c.Env, "HOME="+os.Getenv("REKORTMPDIR")) + } + b, err := c.Output() + if err != nil { + t.Log(string(b)) + t.Fatal(err) + } + return string(b) +} + func runCliErr(t *testing.T, arg ...string) string { t.Helper() arg = append(arg, rekorServerFlag())