From 5d80dd29d6d30a8cd409634db7a48f894473231b Mon Sep 17 00:00:00 2001 From: Andrew Bayer Date: Tue, 16 Jul 2019 12:29:39 -0400 Subject: [PATCH] Add "volume" PipelineResource fixes #1062 This will allow copying content either into or out of a `TaskRun`, either to an existing volume or a newly created volume. The immediate use case is for copying a pipeline's workspace to be made available as the input for another pipeline's workspace without needing to deal with uploading everything to a bucket. The volume, whether already existing or created, will not be deleted at the end of the `PipelineRun`, unlike the artifact storage PVC. This is just the initial work - the unit tests are not complete, and there need to be e2e tests, obviously, but I just wanted to get this initial work up for evaluation. Signed-off-by: Andrew Bayer --- pkg/apis/pipeline/v1alpha1/resource_types.go | 7 + pkg/apis/pipeline/v1alpha1/taskrun_types.go | 7 + pkg/apis/pipeline/v1alpha1/volume_resource.go | 221 ++++++++++++++++++ .../pipeline/v1alpha1/volume_resource_test.go | 158 +++++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 16 ++ .../taskrun/resources/input_resource_test.go | 169 ++++++++++++++ .../taskrun/resources/input_resources.go | 1 + .../taskrun/resources/output_resource.go | 68 ++++++ .../taskrun/resources/output_resource_test.go | 158 +++++++++++++ .../v1alpha1/taskrun/timeout_check_test.go | 6 +- 10 files changed, 808 insertions(+), 3 deletions(-) create mode 100644 pkg/apis/pipeline/v1alpha1/volume_resource.go create mode 100644 pkg/apis/pipeline/v1alpha1/volume_resource_test.go diff --git a/pkg/apis/pipeline/v1alpha1/resource_types.go b/pkg/apis/pipeline/v1alpha1/resource_types.go index f6c50f8d520..0848b299135 100644 --- a/pkg/apis/pipeline/v1alpha1/resource_types.go +++ b/pkg/apis/pipeline/v1alpha1/resource_types.go @@ -46,6 +46,8 @@ const ( // PipelineResourceTypeCloudEvent indicates that this source is a cloud event URI PipelineResourceTypeCloudEvent PipelineResourceType = "cloudEvent" + // PipelineResourceTypeVolume indicates that this source is a PVC. + PipelineResourceTypeVolume PipelineResourceType = "volume" ) // AllResourceTypes can be used for validation to check if a provided Resource type is one of the known types. @@ -147,8 +149,13 @@ func ResourceFromType(r *PipelineResource) (PipelineResourceInterface, error) { return NewStorageResource(r) case PipelineResourceTypePullRequest: return NewPullRequestResource(r) +<<<<<<< HEAD case PipelineResourceTypeCloudEvent: return NewCloudEventResource(r) +======= + case PipelineResourceTypeVolume: + return NewVolumeResource(r) +>>>>>>> Add "volume" PipelineResource } return nil, xerrors.Errorf("%s is an invalid or unimplemented PipelineResource", r.Spec.Type) } diff --git a/pkg/apis/pipeline/v1alpha1/taskrun_types.go b/pkg/apis/pipeline/v1alpha1/taskrun_types.go index 8b88896f7ae..cc66f791121 100644 --- a/pkg/apis/pipeline/v1alpha1/taskrun_types.go +++ b/pkg/apis/pipeline/v1alpha1/taskrun_types.go @@ -285,6 +285,13 @@ func (tr *TaskRun) GetPipelineRunPVCName() string { return "" } +// GetOwnerReference gets the task run as owner reference for any related objects +func (tr *TaskRun) GetOwnerReference() []metav1.OwnerReference { + return []metav1.OwnerReference{ + *metav1.NewControllerRef(tr, groupVersionKind), + } +} + // HasPipelineRunOwnerReference returns true of TaskRun has // owner reference of type PipelineRun func (tr *TaskRun) HasPipelineRunOwnerReference() bool { diff --git a/pkg/apis/pipeline/v1alpha1/volume_resource.go b/pkg/apis/pipeline/v1alpha1/volume_resource.go new file mode 100644 index 00000000000..83d064f7021 --- /dev/null +++ b/pkg/apis/pipeline/v1alpha1/volume_resource.go @@ -0,0 +1,221 @@ +/* + Copyright 2019 The Tekton Authors + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + "path/filepath" + "strings" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/tektoncd/pipeline/pkg/names" + "golang.org/x/xerrors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + // DefaultPvcSize is the default size of the PVC to create + DefaultPvcSize = "5Gi" + + // VolumeMountDir is where the volume resource will be mounted + VolumeMountDir = "/volumeresource" +) + +// VolumeResource is a volume from which to get artifacts which is required +// by a Build/Task for context (e.g. a archive from which to build an image). +type VolumeResource struct { + Name string `json:"name"` + Type PipelineResourceType `json:"type"` + DestinationDir string `json:"destinationDir"` + SourceDir string `json:"sourceDir"` + Size string `json:"size"` +} + +// NewVolumeResource creates a new volume resource to pass to a Task +func NewVolumeResource(r *PipelineResource) (*VolumeResource, error) { + if r.Spec.Type != PipelineResourceTypeVolume { + return nil, xerrors.Errorf("VolumeResource: Cannot create a volume resource from a %s Pipeline Resource", r.Spec.Type) + } + size := DefaultPvcSize + + for _, param := range r.Spec.Params { + switch { + case strings.EqualFold(param.Name, "Size"): + size = param.Value + } + } + + return &VolumeResource{ + Name: r.Name, + Type: r.Spec.Type, + Size: size, + }, nil +} + +// GetName returns the name of the resource +func (s VolumeResource) GetName() string { + return s.Name +} + +// GetType returns the type of the resource, in this case "volume" +func (s VolumeResource) GetType() PipelineResourceType { + return PipelineResourceTypeVolume +} + +// Replacements is used for template replacement on an VolumeResource inside of a Taskrun. +func (s *VolumeResource) Replacements() map[string]string { + return map[string]string{ + "name": s.Name, + "type": string(s.Type), + "path": s.DestinationDir, + } +} + +// SetSourceDirectory sets the source directory at runtime like where is the resource going to be copied from +func (s *VolumeResource) SetSourceDirectory(srcDir string) { s.SourceDir = srcDir } + +// SetDestinationDirectory sets the destination directory at runtime like where is the resource going to be copied to +func (s *VolumeResource) SetDestinationDirectory(destDir string) { s.DestinationDir = destDir } + +// GetVolume returns the volume for this resource, creating it if necessary. +func (s *VolumeResource) GetVolume(c kubernetes.Interface, tr *TaskRun) (*corev1.Volume, error) { + _, err := c.CoreV1().PersistentVolumeClaims(tr.Namespace).Get(s.Name, metav1.GetOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + return nil, xerrors.Errorf("failed to get claim Persistent Volume %q due to error: %w", tr.Name, err) + } + pvcSize, err := resource.ParseQuantity(s.Size) + if err != nil { + return nil, xerrors.Errorf("failed to create Persistent Volume spec for %q due to error: %w", tr.Name, err) + } + pvcSpec := s.GetPVCSpec(tr, pvcSize) + _, err = c.CoreV1().PersistentVolumeClaims(tr.Namespace).Create(pvcSpec) + if err != nil { + return nil, xerrors.Errorf("failed to claim Persistent Volume %q due to error: %w", tr.Name, err) + } + } + + return &corev1.Volume{ + Name: s.Name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: s.Name}, + }, + }, nil +} + +// GetUploadContainerSpec gets container spec for gcs resource to be uploaded like +// set environment variable from secret params and set volume mounts for those secrets +func (s *VolumeResource) GetUploadContainerSpec() ([]corev1.Container, error) { + if s.DestinationDir == "" { + return nil, xerrors.Errorf("VolumeResource: Expect Destination Directory param to be set: %s", s.Name) + } + return []corev1.Container{{ + Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("upload-mkdir-%s", s.Name)), + Image: *BashNoopImage, + Command: []string{"/ko-app/bash"}, + Args: []string{ + "-args", strings.Join([]string{"mkdir", "-p", filepath.Join(VolumeMountDir, s.DestinationDir)}, " "), + }, + VolumeMounts: []corev1.VolumeMount{s.GetPvcMount()}, + }, { + Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("upload-copy-%s", s.Name)), + Image: *BashNoopImage, + Command: []string{"/ko-app/bash"}, + Args: []string{ + "-args", strings.Join([]string{"cp", "-r", fmt.Sprintf("%s/.", s.SourceDir), filepath.Join(VolumeMountDir, s.DestinationDir)}, " "), + }, + VolumeMounts: []corev1.VolumeMount{s.GetPvcMount()}, + }}, nil +} + +// GetDownloadContainerSpec returns an array of container specs to download gcs storage object +func (s *VolumeResource) GetDownloadContainerSpec() ([]corev1.Container, error) { + if s.DestinationDir == "" { + return nil, xerrors.Errorf("VolumeResource: Expect Destination Directory param to be set %s", s.Name) + } + + return []corev1.Container{ + CreateDirContainer(s.Name, s.DestinationDir), { + Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("download-copy-%s", s.Name)), + Image: *BashNoopImage, + Command: []string{"/ko-app/bash"}, + Args: []string{ + "-args", strings.Join([]string{"cp", "-r", fmt.Sprintf("%s/.", filepath.Join(VolumeMountDir, s.SourceDir)), s.DestinationDir}, " "), + }, + VolumeMounts: []corev1.VolumeMount{s.GetPvcMount()}, + }}, nil +} + +// GetPVCSpec returns the PVC to create for a given TaskRun +func (s *VolumeResource) GetPVCSpec(tr *TaskRun, pvcSize resource.Quantity) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: tr.Namespace, + Name: s.Name, + OwnerReferences: tr.GetOwnerReference(), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: pvcSize, + }, + }, + }, + } +} + +// GetPvcMount returns a mounting of the volume with the mount path /volumeresource +func (s *VolumeResource) GetPvcMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: s.Name, // resource pvc name + MountPath: VolumeMountDir, // nothing should be mounted here + } +} + +func (s *VolumeResource) GetDownloadVolumeSpec(spec *TaskSpec) ([]corev1.Volume, error) { + volume, err := s.GetVolume(kubeClient, taskRun) + if err != nil { + return nil, err + } + return []corev1.Volume{*volume}, nil +} + + +func (s * VolumeResource) GetDownloadContainerSpec(spec *TaskSpec) ([]corev1.Container, error) { + var volumeContainers []corev1.Container + + var actualSourcePaths []string + + if len(sourcePaths) > 0 { + actualSourcePaths = append(actualSourcePaths, sourcePaths...) + } else { + actualSourcePaths = append(actualSourcePaths, volumeResource.Name) + } + + for _, src := range actualSourcePaths { + volumeResource.SetSourceDirectory(src) + containers, err := volumeResource.GetDownloadContainerSpec() + if err != nil { + return nil, err + } + volumeContainers = append(volumeContainers, containers...) + } + + return volumeContainers, nil +} \ No newline at end of file diff --git a/pkg/apis/pipeline/v1alpha1/volume_resource_test.go b/pkg/apis/pipeline/v1alpha1/volume_resource_test.go new file mode 100644 index 00000000000..8d805134066 --- /dev/null +++ b/pkg/apis/pipeline/v1alpha1/volume_resource_test.go @@ -0,0 +1,158 @@ +/* + Copyright 2019 The Tekton Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1_test + +import ( + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + tb "github.com/tektoncd/pipeline/test/builder" + "github.com/tektoncd/pipeline/test/names" + corev1 "k8s.io/api/core/v1" + "testing" +) + +func TestNewVolumeResource(t *testing.T) { + for _, c := range []struct { + desc string + resource *v1alpha1.PipelineResource + want *v1alpha1.VolumeResource + }{{ + desc: "basic volume resource", + resource: tb.PipelineResource("test-volume-resource", "default", tb.PipelineResourceSpec( + v1alpha1.PipelineResourceTypeVolume, + tb.PipelineResourceSpecParam("name", "test-volume-resource"), + )), + want: &v1alpha1.VolumeResource{ + Name: "test-volume-resource", + Type: v1alpha1.PipelineResourceTypeVolume, + Size: "5Gi", + }, + }, { + desc: "volume resource with size", + resource: tb.PipelineResource("test-volume-resource", "default", tb.PipelineResourceSpec( + v1alpha1.PipelineResourceTypeVolume, + tb.PipelineResourceSpecParam("name", "test-volume-resource"), + tb.PipelineResourceSpecParam("size", "10Gi"), + )), + want: &v1alpha1.VolumeResource{ + Name: "test-volume-resource", + Type: v1alpha1.PipelineResourceTypeVolume, + Size: "10Gi", + }, + }} { + t.Run(c.desc, func(t *testing.T) { + got, err := v1alpha1.NewVolumeResource(c.resource) + if err != nil { + t.Errorf("Test: %q; TestNewVolumeResource() error = %v", c.desc, err) + } + if d := cmp.Diff(got, c.want); d != "" { + t.Errorf("Diff:\n%s", d) + } + }) + } +} + +func Test_VolumeResource_GetDownloadContainerSpec(t *testing.T) { + names.TestingSeed() + testcases := []struct { + name string + volumeResource *v1alpha1.VolumeResource + wantContainers []corev1.Container + wantErr bool + }{{ + name: "valid volume resource config", + volumeResource: &v1alpha1.VolumeResource{ + Name: "test-volume-resource", + Type: v1alpha1.PipelineResourceTypeVolume, + Size: v1alpha1.DefaultPvcSize, + DestinationDir: "/workspace", + SourceDir: "/src-dir", + }, + wantContainers: []corev1.Container{{ + Name: "create-dir-test-volume-resource-9l9zj", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /workspace"}, + }, { + Name: "download-copy-test-volume-resource-mz4c7", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /volumeresource/src-dir/. /workspace"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "test-volume-resource", + MountPath: "/volumeresource", + }}, + }}, + }} + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + gotContainers, err := tc.volumeResource.GetDownloadContainerSpec() + if tc.wantErr && err == nil { + t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err) + } + if d := cmp.Diff(gotContainers, tc.wantContainers); d != "" { + t.Errorf("Error mismatch between download containers spec: %s", d) + } + }) + } +} + +func Test_VolumeResource_GetUploadContainerSpec(t *testing.T) { + names.TestingSeed() + testcases := []struct { + name string + volumeResource *v1alpha1.VolumeResource + wantContainers []corev1.Container + wantErr bool + }{{ + name: "valid volume resource config", + volumeResource: &v1alpha1.VolumeResource{ + Name: "test-volume-resource", + Type: v1alpha1.PipelineResourceTypeVolume, + Size: v1alpha1.DefaultPvcSize, + DestinationDir: "/workspace", + SourceDir: "/src-dir", + }, + wantContainers: []corev1.Container{{ + Name: "upload-mkdir-test-volume-resource-9l9zj", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /volumeresource/workspace"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "test-volume-resource", + MountPath: "/volumeresource", + }}, + }, { + Name: "upload-copy-test-volume-resource-mz4c7", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /src-dir/. /volumeresource/workspace"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "test-volume-resource", + MountPath: "/volumeresource", + }}, + }}, + }} + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + gotContainers, err := tc.volumeResource.GetUploadContainerSpec() + if tc.wantErr && err == nil { + t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err) + } + if d := cmp.Diff(gotContainers, tc.wantContainers); d != "" { + t.Errorf("Error mismatch between download containers spec: %s", d) + } + }) + } +} diff --git a/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go index 7e53c3174ae..0c2828331d8 100644 --- a/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go @@ -1829,3 +1829,19 @@ func (in *TestResult) DeepCopy() *TestResult { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeResource) DeepCopyInto(out *VolumeResource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeResource. +func (in *VolumeResource) DeepCopy() *VolumeResource { + if in == nil { + return nil + } + out := new(VolumeResource) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go index 022e881e466..53d940fc190 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go @@ -61,6 +61,13 @@ var ( Type: "cluster", }}, } + volumeInputs = &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "workspace", + Type: "volume", + TargetPath: "sub-dir", + }}, + } ) func setUp(t *testing.T) { @@ -189,6 +196,18 @@ func setUp(t *testing.T) { Value: "non-existent", }}, }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "volume-valid", + Namespace: "marshmallow", + }, + Spec: v1alpha1.PipelineResourceSpec{ + Type: "volume", + Params: []v1alpha1.Param{{ + Name: "Size", + Value: "10Gi", + }}, + }, }} inputResourceInterfaces = make(map[string]v1alpha1.PipelineResourceInterface) for _, r := range rs { @@ -226,6 +245,15 @@ func TestAddResourceToTask(t *testing.T) { Inputs: gcsInputs, }, } + taskWithVolume := &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Name: "task-with-volume", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: volumeInputs, + }, + } taskRun := &v1alpha1.TaskRun{ ObjectMeta: metav1.ObjectMeta{ @@ -690,6 +718,147 @@ func TestAddResourceToTask(t *testing.T) { }}, }}, }, + }, { + desc: "volume resource as input with paths", + task: taskWithVolume, + taskRun: &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "get-from-volume", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskRunSpec{ + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskResourceBinding{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "volume-valid", + }, + Name: "workspace", + Paths: []string{"workspace"}, + }}, + }, + }, + }, + wantErr: false, + want: &v1alpha1.TaskSpec{ + Inputs: volumeInputs, + Steps: []corev1.Container{{ + Name: "create-dir-volume-valid-9l9zj", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /workspace/sub-dir"}, + }, { + Name: "download-copy-volume-valid-mz4c7", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /volumeresource/workspace/. /workspace/sub-dir"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "volume-valid", + MountPath: "/volumeresource", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "volume-valid", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "volume-valid", + ReadOnly: false, + }, + }, + }}, + }, + }, { + desc: "volume resource as input without paths", + task: taskWithVolume, + taskRun: &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "get-from-volume", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskRunSpec{ + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskResourceBinding{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "volume-valid", + }, + Name: "workspace", + }}, + }, + }, + }, + wantErr: false, + want: &v1alpha1.TaskSpec{ + Inputs: volumeInputs, + Steps: []corev1.Container{{ + Name: "create-dir-volume-valid-9l9zj", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /workspace/sub-dir"}, + }, { + Name: "download-copy-volume-valid-mz4c7", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /volumeresource/volume-valid/. /workspace/sub-dir"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "volume-valid", + MountPath: "/volumeresource", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "volume-valid", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "volume-valid", + ReadOnly: false, + }, + }, + }}, + }, + }, { + desc: "volume resource as input from previous task", + task: taskWithVolume, + taskRun: &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "get-from-volume", + Namespace: "marshmallow", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "PipelineRun", + Name: "pipelinerun", + }}, + }, + Spec: v1alpha1.TaskRunSpec{ + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskResourceBinding{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "volume-valid", + }, + Name: "workspace", + Paths: []string{"prev-task-path"}, + }}, + }, + }, + }, + wantErr: false, + want: &v1alpha1.TaskSpec{ + Inputs: volumeInputs, + Steps: []corev1.Container{{ + Name: "create-dir-workspace-mz4c7", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /workspace/sub-dir"}, + }, { + Name: "source-copy-workspace-9l9zj", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r prev-task-path/. /workspace/sub-dir"}, + VolumeMounts: []corev1.VolumeMount{{MountPath: "/pvc", Name: "pipelinerun-pvc"}}, + }}, + Volumes: []corev1.Volume{{ + Name: "pipelinerun-pvc", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pipelinerun-pvc"}, + }, + }}, + }, }} { t.Run(c.desc, func(t *testing.T) { setUp(t) diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go b/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go index d80e2e521b1..4726149d563 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go @@ -118,6 +118,7 @@ func AddInputResource( resourceVolumes, err = resource.GetDownloadVolumeSpec(taskSpec) if err != nil { return nil, xerrors.Errorf("task %q invalid resource download spec: %q; error %w", taskName, boundResource.ResourceRef.Name, err) + } allResourceContainers = append(allResourceContainers, resourceContainers...) diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go index ed4bcd1d57e..34027eb1afb 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go @@ -35,6 +35,7 @@ var ( allowedOutputResources = map[v1alpha1.PipelineResourceType]bool{ v1alpha1.PipelineResourceTypeStorage: true, v1alpha1.PipelineResourceTypeGit: true, + v1alpha1.PipelineResourceTypeVolume: true, } ) @@ -151,3 +152,70 @@ func AddOutputResources( } return taskSpec, nil } +<<<<<<< HEAD +======= + +func addStoreUploadStep(spec *v1alpha1.TaskSpec, + storageResource v1alpha1.PipelineStorageResourceInterface, +) ([]corev1.Container, []corev1.Volume, error) { + + gcsContainers, err := storageResource.GetUploadContainerSpec() + if err != nil { + return nil, nil, err + } + var storageVol []corev1.Volume + mountedSecrets := map[string]string{} + + for _, volume := range spec.Volumes { + mountedSecrets[volume.Name] = "" + } + + // Map holds list of secrets that are mounted as volumes + for _, secretParam := range storageResource.GetSecretParams() { + volName := fmt.Sprintf("volume-%s-%s", storageResource.GetName(), secretParam.SecretName) + + gcsSecretVolume := corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretParam.SecretName, + }, + }, + } + + if _, ok := mountedSecrets[volName]; !ok { + storageVol = append(storageVol, gcsSecretVolume) + mountedSecrets[volName] = "" + } + } + return gcsContainers, storageVol, nil +} + +func addVolumeUploadStep(kubeClient kubernetes.Interface, taskRun *v1alpha1.TaskRun, volumeResource *v1alpha1.VolumeResource, sourcePath string, destinations []string) ([]corev1.Container, []corev1.Volume, error) { + var volumeContainers []corev1.Container + + volumeResource.SetSourceDirectory(sourcePath) + + var actualDestinations []string + if len(destinations) > 0 { + actualDestinations = append(actualDestinations, destinations...) + } else { + actualDestinations = append(actualDestinations, volumeResource.Name) + } + + for _, dest := range actualDestinations { + volumeResource.SetDestinationDirectory(dest) + containers, err := volumeResource.GetUploadContainerSpec() + if err != nil { + return nil, nil, err + } + volumeContainers = append(volumeContainers, containers...) + } + + volume, err := volumeResource.GetVolume(kubeClient, taskRun) + if err != nil { + return nil, nil, err + } + return volumeContainers, []corev1.Volume{*volume}, nil +} +>>>>>>> Add "volume" PipelineResource diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go index 4a78af49e9a..d8606c6d78c 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go @@ -89,6 +89,18 @@ func outputResourceSetup(t *testing.T) { Spec: v1alpha1.PipelineResourceSpec{ Type: "image", }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "source-volume", + Namespace: "marshmallow", + }, + Spec: v1alpha1.PipelineResourceSpec{ + Type: "volume", + Params: []v1alpha1.Param{{ + Name: "Size", + Value: "10Gi", + }}, + }, }} outputResources = make(map[string]v1alpha1.PipelineResourceInterface) @@ -749,6 +761,152 @@ func TestValidOutputResources(t *testing.T) { Args: []string{"-args", "mkdir -p /workspace/output/source-workspace"}, }, }, + }, { + name: "volume resource as output with no owner", + desc: "volume resource defined only in output without pipelinerun reference", + taskRun: &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-taskrun-run-only-output-step", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskRunSpec{ + Outputs: v1alpha1.TaskRunOutputs{ + Resources: []v1alpha1.TaskResourceBinding{{ + Name: "source-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-volume", + }, + }}, + }, + }, + }, + task: &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Name: "task1", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskSpec{ + Outputs: &v1alpha1.Outputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "source-workspace", + Type: "volume", + TargetPath: "/workspace", + }}, + }, + }, + }, + wantSteps: []corev1.Container{{ + Name: "upload-mkdir-source-volume-9l9zj", + Image: "override-with-bash-noop:latest", + VolumeMounts: []corev1.VolumeMount{{ + Name: "source-volume", MountPath: "/volumeresource", + }}, + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /volumeresource/source-volume"}, + }, { + Name: "upload-copy-source-volume-mz4c7", + Image: "override-with-bash-noop:latest", + VolumeMounts: []corev1.VolumeMount{{ + Name: "source-volume", MountPath: "/volumeresource", + }}, + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /workspace/. /volumeresource/source-volume"}, + }}, + wantVolumes: []corev1.Volume{{ + Name: "source-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "source-volume", ReadOnly: false}, + }, + }}, + }, { + name: "volume resource as both input and output", + desc: "volume resource defined in both input and output", + taskRun: &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-taskrun-run-output-steps", + Namespace: "marshmallow", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "PipelineRun", + Name: "pipelinerun-parent", + }}, + }, + Spec: v1alpha1.TaskRunSpec{ + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskResourceBinding{{ + Name: "source-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-volume", + }, + }}, + }, + Outputs: v1alpha1.TaskRunOutputs{ + Resources: []v1alpha1.TaskResourceBinding{{ + Name: "source-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-volume", + }, + Paths: []string{"pipeline-task-path"}, + }}, + }, + }, + }, + task: &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Name: "task1", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "source-workspace", + Type: "volume", + TargetPath: "faraway-disk", + }}, + }, + Outputs: &v1alpha1.Outputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "source-workspace", + Type: "volume", + }}, + }, + }, + }, + wantSteps: []corev1.Container{{ + Name: "upload-mkdir-source-volume-9l9zj", + Image: "override-with-bash-noop:latest", + VolumeMounts: []corev1.VolumeMount{{ + Name: "source-volume", MountPath: "/volumeresource", + }}, + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p /volumeresource/pipeline-task-path"}, + }, { + Name: "upload-copy-source-volume-mz4c7", + Image: "override-with-bash-noop:latest", + VolumeMounts: []corev1.VolumeMount{{ + Name: "source-volume", + MountPath: "/volumeresource", + }}, + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /workspace/faraway-disk/. /volumeresource/pipeline-task-path"}, + }, { + Name: "source-mkdir-source-volume-mssqb", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "mkdir -p pipeline-task-path"}, + VolumeMounts: []corev1.VolumeMount{{Name: "pipelinerun-parent-pvc", MountPath: "/pvc"}}, + }, { + Name: "source-copy-source-volume-78c5n", + Image: "override-with-bash-noop:latest", + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", "cp -r /workspace/faraway-disk/. pipeline-task-path"}, + VolumeMounts: []corev1.VolumeMount{{Name: "pipelinerun-parent-pvc", MountPath: "/pvc"}}, + }}, + wantVolumes: []corev1.Volume{{ + Name: "source-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "source-volume", ReadOnly: false}, + }, + }}, }} { t.Run(c.name, func(t *testing.T) { names.TestingSeed() diff --git a/pkg/reconciler/v1alpha1/taskrun/timeout_check_test.go b/pkg/reconciler/v1alpha1/taskrun/timeout_check_test.go index 73b77ef5145..e17415a8978 100644 --- a/pkg/reconciler/v1alpha1/taskrun/timeout_check_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/timeout_check_test.go @@ -37,19 +37,19 @@ func TestCheckTimeout(t *testing.T) { name: "TaskRun not started", taskRun: tb.TaskRun("test-taskrun-not-started", "foo", tb.TaskRunSpec( tb.TaskRunTaskRef(simpleTask.Name), - ), tb.TaskRunStatus(tb.StatusCondition(apis.Condition{}), tb.TaskRunStartTime(zeroTime))), + ), tb.TaskRunStatus(tb.Condition(apis.Condition{}), tb.TaskRunStartTime(zeroTime))), expectedStatus: false, }, { name: "TaskRun no timeout", taskRun: tb.TaskRun("test-taskrun-no-timeout", "foo", tb.TaskRunSpec( tb.TaskRunTaskRef(simpleTask.Name), tb.TaskRunTimeout(0), - ), tb.TaskRunStatus(tb.StatusCondition(apis.Condition{}), tb.TaskRunStartTime(time.Now().Add(-15*time.Hour)))), + ), tb.TaskRunStatus(tb.Condition(apis.Condition{}), tb.TaskRunStartTime(time.Now().Add(-15*time.Hour)))), expectedStatus: false, }, { name: "TaskRun timed out", taskRun: tb.TaskRun("test-taskrun-timeout", "foo", tb.TaskRunSpec( tb.TaskRunTaskRef(simpleTask.Name), tb.TaskRunTimeout(10*time.Second), - ), tb.TaskRunStatus(tb.StatusCondition(apis.Condition{}), tb.TaskRunStartTime(time.Now().Add(-15*time.Second)))), + ), tb.TaskRunStatus(tb.Condition(apis.Condition{}), tb.TaskRunStartTime(time.Now().Add(-15*time.Second)))), expectedStatus: true, }}