Add "volume" PipelineResource

This will allow copying content either into or out of a `TaskRun`,
either to an existing volume or a newly created volume. The immediate
use case is for copying a pipeline's workspace to be made available as
the input for another pipeline's workspace without needing to deal
with uploading everything to a bucket. The volume, whether already
existing or created, will not be deleted at the end of the
`PipelineRun`, unlike the artifact storage PVC.

The Volume resource is a sub-type of the general Storage resource.

Since this type will require the creation of a PVC to function (to be
configurable later), this commit adds a Setup interface that
PipelineResources can implement if they need to do setup that involves
instantiating objects in Kube. This could be a place to later add
features like caching, and also is the sort of design we'd expect once
PipelineResources are extensible (PipelineResources will be free to do
whatever setup they need).

fixes tektoncd#1062

Co-authored-by: Dan Lorenc <lorenc.d@gmail.com>
Co-authored-by: Christie Wilson <bobcatfish@gmail.com>

pkg/apis/pipeline/v1alpha1/build_gcs_resource.go

@@ -72,6 +72,9 @@ type BuildGCSResource struct {
 	ArtifactType GCSArtifactType
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that does nothing because no setup is needed.
+func (s BuildGCSResource) GetSetup() PipelineResourceSetupInterface { return &NoSetup{} }
+
 // NewBuildGCSResource creates a new BuildGCS resource to pass to a Task
 func NewBuildGCSResource(r *PipelineResource) (*BuildGCSResource, error) {
 	if r.Spec.Type != PipelineResourceTypeStorage {

pkg/apis/pipeline/v1alpha1/build_gcs_resource_test.go

@@ -78,7 +78,7 @@ func Test_Invalid_BuildGCSResource(t *testing.T) {
 		)),
 	}} {
 		t.Run(tc.name, func(t *testing.T) {
-			_, err := v1alpha1.NewStorageResource(tc.pipelineResource)
+			_, err := v1alpha1.NewBuildGCSResource(tc.pipelineResource)
 			if err == nil {
 				t.Error("Expected error creating BuildGCS resource")
 			}

pkg/apis/pipeline/v1alpha1/cloud_event_resource.go

@@ -61,6 +61,9 @@ func NewCloudEventResource(r *PipelineResource) (*CloudEventResource, error) {
 	}, nil
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that can create the backing PVC if needed.
+func (s CloudEventResource) GetSetup() PipelineResourceSetupInterface { return SetupPVC{} }
+
 // GetName returns the name of the resource
 func (s CloudEventResource) GetName() string {
 	return s.Name

pkg/apis/pipeline/v1alpha1/cluster_resource.go

@@ -57,6 +57,9 @@ type ClusterResource struct {
 	Secrets []SecretParam `json:"secrets"`
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that does nothing because no setup is needed.
+func (s ClusterResource) GetSetup() PipelineResourceSetupInterface { return &NoSetup{} }
+
 // NewClusterResource create a new k8s cluster resource to pass to a pipeline task
 func NewClusterResource(r *PipelineResource) (*ClusterResource, error) {
 	if r.Spec.Type != PipelineResourceTypeCluster {

pkg/apis/pipeline/v1alpha1/gcs_resource.go

@@ -43,6 +43,9 @@ type GCSResource struct {
 	Secrets []SecretParam `json:"secrets"`
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that does nothing because no setup is needed.
+func (s GCSResource) GetSetup() PipelineResourceSetupInterface { return &NoSetup{} }
+
 // NewGCSResource creates a new GCS resource to pass to a Task
 func NewGCSResource(r *PipelineResource) (*GCSResource, error) {
 	if r.Spec.Type != PipelineResourceTypeStorage {

pkg/apis/pipeline/v1alpha1/git_resource.go

@@ -46,6 +46,9 @@ type GitResource struct {
 	Revision string `json:"revision"`
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that does nothing because no setup is needed.
+func (s GitResource) GetSetup() PipelineResourceSetupInterface { return &NoSetup{} }
+
 // NewGitResource creates a new git resource to pass to a Task
 func NewGitResource(r *PipelineResource) (*GitResource, error) {
 	if r.Spec.Type != PipelineResourceTypeGit {

pkg/apis/pipeline/v1alpha1/image_resource.go

@@ -55,6 +55,9 @@ type ImageResource struct {
 	OutputImageDir string
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that does nothing because no setup is needed.
+func (s ImageResource) GetSetup() PipelineResourceSetupInterface { return &NoSetup{} }
+
 // GetName returns the name of the resource
 func (s ImageResource) GetName() string {
 	return s.Name

pkg/apis/pipeline/v1alpha1/pull_request_resource.go

@@ -49,6 +49,9 @@ type PullRequestResource struct {
 	Secrets []SecretParam `json:"secrets"`
 }
 
+// GetSetup returns a PipelineResourceSetupInterface that does nothing because no setup is needed.
+func (s PullRequestResource) GetSetup() PipelineResourceSetupInterface { return &NoSetup{} }
+
 // NewPullRequestResource create a new git resource to pass to a Task
 func NewPullRequestResource(r *PipelineResource) (*PullRequestResource, error) {
 	if r.Spec.Type != PipelineResourceTypePullRequest {

pkg/apis/pipeline/v1alpha1/resource_types.go

@@ -20,6 +20,7 @@ import (
 	"golang.org/x/xerrors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"knative.dev/pkg/apis"
 )
 
@@ -53,13 +54,52 @@ var AllResourceTypes = []PipelineResourceType{PipelineResourceTypeGit, PipelineR
 
 // PipelineResourceInterface interface to be implemented by different PipelineResource types
 type PipelineResourceInterface interface {
+	// GetName returns the name of this PipelineResource instnace
 	GetName() string
+	// GetType returns the type of this PipelineResource (often a super type)
 	GetType() PipelineResourceType
+	// Replacements returns all the attributes that this PipelineResource has that
+	// can be used for variable replacement.
 	Replacements() map[string]string
+	// GetDownloadSteps returns the steps that should be added to the TaskRun execution
+	// before the user requested steps in order to initialize the PipelineResource on disk.
 	GetDownloadSteps(sourcePath string) ([]Step, error)
+	// GetUploadSteps returns the steps that should be added to the TaskRun execution
+	// after the user requested steps in order to update the PipelineResource from the state
+	// on disk.
 	GetUploadSteps(sourcePath string) ([]Step, error)
+	// GetUploadVolumeSpec returns the volume spec that should be added to the TaskRun
+	// execution if the PipelineResource is used as an output and the PipelineResource requires
+	// a volume.
 	GetUploadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error)
+	// GetDownloadVolumeSpec returns the volume spec that should be added to the TaskRun
+	// execution if the PipelineResource is used as an input and the PipelineResource requires
+	// a volume.
 	GetDownloadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error)
+	// GetSetup returns the instance of PipelineResourceSetupInterface that the PipelineResource
+	// needs in order to perform operations before it can be realized. This function should be
+	// idempotent. NoSetup can be used by PipelineResources that do not require setup.
+	GetSetup() PipelineResourceSetupInterface
+}
+
+// PipelineResourceSetupInterface is an interface that can be implemented by objects that know
+// how to perform setup required by PipelineResources before they can be realized. PipelineResources
+// can return the instance of the appropriate PipelineResourceSetupInterface.
+type PipelineResourceSetupInterface interface {
+	// Setup is called to setup any state that is required by a PipelineResource before
+	// executing. It is provided with a kubernetes clientset c so that it can make changes
+	// in the outside world if required, the owner references o that it should
+	// add to any new kubernetes objects it instantiates, and the PipelineResourceInterface r.
+	Setup(r PipelineResourceInterface, o []metav1.OwnerReference, c kubernetes.Interface) error
+}
+
+// NoSetup is a PipelineResourceSetupInterface that doesn't do anything. It can be used by
+// PipelineResources that do not require any setup.
+type NoSetup struct{}
+
+// Setup for a NoSetup object does nothing, indicating that no setup is required.
+func (n *NoSetup) Setup(r PipelineResourceInterface, o []metav1.OwnerReference, c kubernetes.Interface) error {
+	return nil
 }
 
 // SecretParam indicates which secret can be used to populate a field of the resource
@@ -152,7 +192,9 @@ type ResourceDeclaration struct {
 	TargetPath string `json:"targetPath,omitempty"`
 }
 
-// ResourceFromType returns a PipelineResourceInterface from a PipelineResource's type.
+// ResourceFromType returns an instance of the correct PipelineResource object type which can be
+// used to add input and ouput containers as well as volumes to a TaskRun's pod in order to realize
+// a PipelineResource in a pod.
 func ResourceFromType(r *PipelineResource) (PipelineResourceInterface, error) {
 	switch r.Spec.Type {
 	case PipelineResourceTypeGit:

pkg/apis/pipeline/v1alpha1/storage_resource.go

@@ -24,20 +24,29 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
+// PipelineResourceStorageType is used as an enum for subtypes of the storage resource.
 type PipelineResourceStorageType string
 
 const (
-	// PipelineResourceTypeGCS indicates that resource source is a GCS blob/directory.
-	PipelineResourceTypeGCS      PipelineResourceType = "gcs"
+	// PipelineResourceTypeGCS is the subtype for the GCSResources, which is backed by a GCS blob/directory.
+	PipelineResourceTypeGCS PipelineResourceType = "gcs"
+	// PipelineResourceTypeBuildGCS is the subtype for the BuildGCSResources, which is simialr to the GCSResource but
+	// with additional funcitonality that was added to be compatible with knative build.
 	PipelineResourceTypeBuildGCS PipelineResourceType = "build-gcs"
+	// PipelineResourceTypeVolume is the subtype for the VolumeResource, which is backed by a PVC.
+	PipelineResourceTypeVolume PipelineResourceType = "volume"
 )
 
-// PipelineResourceInterface interface to be implemented by different PipelineResource types
+// PipelineStorageResourceInterface is the interface for subtypes of the storage type.
+// It adds a function to the PipelineResourceInterface for retrieving secrets that are usually
+// needed for storage PipelineResources.
 type PipelineStorageResourceInterface interface {
 	PipelineResourceInterface
 	GetSecretParams() []SecretParam
 }
 
+// NewStorageResource returns an instance of the requested storage subtype, which can be used
+// to add input and output steps and volumes to an executing pod.
 func NewStorageResource(r *PipelineResource) (PipelineStorageResourceInterface, error) {
 	if r.Spec.Type != PipelineResourceTypeStorage {
 		return nil, xerrors.Errorf("StoreResource: Cannot create a storage resource from a %s Pipeline Resource", r.Spec.Type)
@@ -50,6 +59,8 @@ func NewStorageResource(r *PipelineResource) (PipelineStorageResourceInterface,
 				return NewGCSResource(r)
 			case strings.EqualFold(param.Value, string(PipelineResourceTypeBuildGCS)):
 				return NewBuildGCSResource(r)
+			case strings.EqualFold(param.Value, string(PipelineResourceTypeVolume)):
+				return NewVolumeResource(r)
 			default:
 				return nil, xerrors.Errorf("%s is an invalid or unimplemented PipelineStorageResource", param.Value)
 			}

pkg/apis/pipeline/v1alpha1/taskrun_types.go

@@ -258,6 +258,13 @@ func (tr *TaskRun) GetPipelineRunPVCName() string {
 	return ""
 }
 
+// GetOwnerReference gets the task run as owner reference for any related objects
+func (tr *TaskRun) GetOwnerReference() []metav1.OwnerReference {
+	return []metav1.OwnerReference{
+		*metav1.NewControllerRef(tr, groupVersionKind),
+	}
+}
+
 // HasPipelineRunOwnerReference returns true of TaskRun has
 // owner reference of type PipelineRun
 func (tr *TaskRun) HasPipelineRunOwnerReference() bool {