From aee2c809d621a032b6f51c622ff521ed30efb82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A1s=20J=C3=A1ky?= Date: Wed, 6 Mar 2024 12:57:19 +0100 Subject: [PATCH] feat: configure Azure provider with task options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: András Jáky --- provider/v2/azure/option.go | 50 +++++ provider/v2/azure/provider.go | 21 +- provider/v2/azure/scanner/scanner.go | 184 +---------------- provider/v2/azure/scanner/task.go | 298 +++++++++++++++++++++++++++ 4 files changed, 373 insertions(+), 180 deletions(-) create mode 100644 provider/v2/azure/option.go create mode 100644 provider/v2/azure/scanner/task.go diff --git a/provider/v2/azure/option.go b/provider/v2/azure/option.go new file mode 100644 index 0000000000..3fb09c0f66 --- /dev/null +++ b/provider/v2/azure/option.go @@ -0,0 +1,50 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +type Option func(*Provider) + +type ScannerTaskOption Option + +func WithEnsureVMInfo(deps ...string) ScannerTaskOption { + return func(s *Provider) { + s.EnsureAssetVMInfo(deps) + } +} + +func WithEnsureSnapshot(deps ...string) ScannerTaskOption { + return func(s *Provider) { + s.EnsureSnapshotWithCleanup(deps) + } +} + +func WithEnsureDisk(deps ...string) ScannerTaskOption { + return func(s *Provider) { + s.EnsureDiskWithCleanup(deps) + } +} + +func WithEnsureScannerVM(deps ...string) ScannerTaskOption { + return func(s *Provider) { + s.EnsureScannerVMWithCleanup(deps) + } +} + +func WithEnsureAttachDiskToScannerVM(deps ...string) ScannerTaskOption { + return func(s *Provider) { + s.EnsureAttachDiskToScannerVM(deps) + } +} diff --git a/provider/v2/azure/provider.go b/provider/v2/azure/provider.go index 0ab62b956c..5b9e5f193f 100644 --- a/provider/v2/azure/provider.go +++ b/provider/v2/azure/provider.go @@ -39,7 +39,7 @@ func (p *Provider) Kind() apitypes.CloudProvider { return apitypes.Azure } -func New(_ context.Context) (*Provider, error) { +func New(_ context.Context, opts ...Option) (*Provider, error) { config, err := NewConfig() if err != nil { return nil, fmt.Errorf("failed to load configuration: %w", err) @@ -65,7 +65,7 @@ func New(_ context.Context) (*Provider, error) { return nil, fmt.Errorf("failed to create compute client factory: %w", err) } - return &Provider{ + provider := &Provider{ Discoverer: &discoverer.Discoverer{ VMClient: computeClientFactory.NewVirtualMachinesClient(), DisksClient: computeClientFactory.NewDisksClient(), @@ -92,5 +92,20 @@ func New(_ context.Context) (*Provider, error) { ScannerStorageContainerName: config.ScannerStorageContainerName, }, Estimator: &estimator.Estimator{}, - }, nil + } + + for _, opt := range opts { + opt(provider) + } + + // default to running all tasks if no options are provided + if len(opts) == 0 { + provider.EnsureAssetVMInfo(nil) + provider.EnsureScannerVMWithCleanup(nil) + provider.EnsureSnapshotWithCleanup([]string{scanner.EnsureAssetVMInfoTaskName}) + provider.EnsureDiskWithCleanup([]string{scanner.EnsureAssetVMInfoTaskName, scanner.EnsureSnapshotTaskName}) + provider.EnsureAttachDiskToScannerVM([]string{scanner.EnsureDiskTaskName, scanner.EnsureScannerVMTaskName}) + } + + return provider, nil } diff --git a/provider/v2/azure/scanner/scanner.go b/provider/v2/azure/scanner/scanner.go index 654aac3472..7134de48a9 100644 --- a/provider/v2/azure/scanner/scanner.go +++ b/provider/v2/azure/scanner/scanner.go @@ -19,14 +19,13 @@ package scanner import ( "context" "fmt" - "strings" + "sync" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" "github.com/openclarity/vmclarity/provider" - "github.com/openclarity/vmclarity/provider/v2/azure/utils" "github.com/openclarity/vmclarity/workflow" workflowTypes "github.com/openclarity/vmclarity/workflow/types" ) @@ -58,113 +57,19 @@ type Scanner struct { ScannerSecurityGroup string ScannerStorageAccountName string ScannerStorageContainerName string -} -type AssetScanState struct { - assetVM armcompute.VirtualMachinesClientGetResponse - scannerVM armcompute.VirtualMachine - snapshot armcompute.Snapshot - disk armcompute.Disk + RunAssetScanTasks []*workflowTypes.Task[*AssetScanState] + RemoveAssetScanTasks []*workflowTypes.Task[*AssetScanState] } // nolint:cyclop func (s *Scanner) RunAssetScan(ctx context.Context, config *provider.ScanJobConfig) error { - tasks := []*workflowTypes.Task[*AssetScanState]{ - { - Name: "GetVMInfo", - Deps: nil, - Fn: func(ctx context.Context, state *AssetScanState) error { - vmInfo, err := config.AssetInfo.AsVMInfo() - if err != nil { - return provider.FatalErrorf("unable to get vminfo from asset: %w", err) - } - - resourceGroup, vmName, err := resourceGroupAndNameFromInstanceID(vmInfo.InstanceID) - if err != nil { - return err - } - - state.assetVM, err = s.VMClient.Get(ctx, resourceGroup, vmName, nil) - if err != nil { - _, err = utils.HandleAzureRequestError(err, "getting asset virtual machine %s", vmName) - return err - } - - return nil - }, - }, - { - Name: "EnsureSnapshot", - Deps: []string{"GetVMInfo"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - var err error - - state.snapshot, err = s.ensureSnapshotForVMRootVolume(ctx, config, state.assetVM.VirtualMachine) - if err != nil { - return fmt.Errorf("failed to ensure snapshot for vm root volume: %w", err) - } - - return nil - }, - }, - { - Name: "EnsureDisk", - Deps: []string{"GetVMInfo", "EnsureSnapshot"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - var err error - - if *state.assetVM.Location == s.ScannerLocation { - state.disk, err = s.ensureManagedDiskFromSnapshot(ctx, config, state.snapshot) - if err != nil { - return fmt.Errorf("failed to ensure managed disk created from snapshot: %w", err) - } - } else { - state.disk, err = s.ensureManagedDiskFromSnapshotInDifferentRegion(ctx, config, state.snapshot) - if err != nil { - return fmt.Errorf("failed to ensure managed disk from snapshot in different region: %w", err) - } - } - - return nil - }, - }, - { - Name: "EnsureScannerVM", - Deps: []string{"EnsureDisk"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - networkInterface, err := s.ensureNetworkInterface(ctx, config) - if err != nil { - return fmt.Errorf("failed to ensure scanner network interface: %w", err) - } - - state.scannerVM, err = s.ensureScannerVirtualMachine(ctx, config, networkInterface) - if err != nil { - return fmt.Errorf("failed to ensure scanner virtual machine: %w", err) - } - - return nil - }, - }, - { - Name: "AttachDiskToScannerVM", - Deps: []string{"EnsureScannerVM", "EnsureDisk"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - err := s.ensureDiskAttachedToScannerVM(ctx, state.scannerVM, state.disk) - if err != nil { - return fmt.Errorf("failed to ensure asset disk is attached to virtual machine: %w", err) - } - - return nil - }, - }, - } - - workflow, err := workflow.New[*AssetScanState, *workflowTypes.Task[*AssetScanState]](tasks) + workflow, err := workflow.New[*AssetScanState, *workflowTypes.Task[*AssetScanState]](s.RunAssetScanTasks) if err != nil { return fmt.Errorf("failed to create RunAssetScan workflow: %w", err) } - err = workflow.Run(ctx, &AssetScanState{}) + err = workflow.Run(ctx, &AssetScanState{config: config, mu: &sync.RWMutex{}}) if err != nil { return fmt.Errorf("failed to run RunAssetScan workflow: %w", err) } @@ -173,90 +78,15 @@ func (s *Scanner) RunAssetScan(ctx context.Context, config *provider.ScanJobConf } func (s *Scanner) RemoveAssetScan(ctx context.Context, config *provider.ScanJobConfig) error { - tasks := []*workflowTypes.Task[*AssetScanState]{ - { - Name: "EnsureScannerVMDeleted", - Deps: nil, - Fn: func(ctx context.Context, state *AssetScanState) error { - err := s.ensureScannerVirtualMachineDeleted(ctx, config) - if err != nil { - return fmt.Errorf("failed to ensure scanner virtual machine deleted: %w", err) - } - return nil - }, - }, - { - Name: "EnsureNetworkInterfaceDeleted", - Deps: nil, // []string{"EnsureScannerVMDeleted"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - err := s.ensureNetworkInterfaceDeleted(ctx, config) - if err != nil { - return fmt.Errorf("failed to ensure network interface deleted: %w", err) - } - - return nil - }, - }, - { - Name: "EnsureTargetDiskDeleted", - Deps: nil, // []string{"EnsureNetworkInterfaceDeleted"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - err := s.ensureTargetDiskDeleted(ctx, config) - if err != nil { - return fmt.Errorf("failed to ensure asset disk deleted: %w", err) - } - - return nil - }, - }, - { - Name: "EnsureBlobDeleted", - Deps: nil, // []string{"EnsureTargetDiskDeleted"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - err := s.ensureBlobDeleted(ctx, config) - if err != nil { - return fmt.Errorf("failed to ensure snapshot copy blob deleted: %w", err) - } - - return nil - }, - }, - { - Name: "EnsureSnapshotDeleted", - Deps: nil, // []string{"EnsureBlobDeleted"}, - Fn: func(ctx context.Context, state *AssetScanState) error { - err := s.ensureSnapshotDeleted(ctx, config) - if err != nil { - return fmt.Errorf("failed to ensure snapshot deleted: %w", err) - } - - return nil - }, - }, - } - - workflow, err := workflow.New[*AssetScanState, *workflowTypes.Task[*AssetScanState]](tasks) + workflow, err := workflow.New[*AssetScanState, *workflowTypes.Task[*AssetScanState]](s.RemoveAssetScanTasks) if err != nil { return fmt.Errorf("failed to create RemoveAssetScan workflow: %w", err) } - err = workflow.Run(ctx, &AssetScanState{}) + err = workflow.Run(ctx, &AssetScanState{config: config, mu: &sync.RWMutex{}}) if err != nil { return fmt.Errorf("failed to run RemoveAssetScan workflow: %w", err) } return nil } - -// Example Instance ID: -// -// /subscriptions/ecad88af-09d5-4725-8d80-906e51fddf02/resourceGroups/vmclarity-sambetts-dev/providers/Microsoft.Compute/virtualMachines/vmclarity-server -// -// Will return "vmclarity-sambetts-dev" and "vmclarity-server". -func resourceGroupAndNameFromInstanceID(instanceID string) (string, string, error) { - idParts := strings.Split(instanceID, "/") - if len(idParts) != instanceIDPartsLength { - return "", "", provider.FatalErrorf("asset instance id in unexpected format got: %s", idParts) - } - return idParts[resourceGroupPartIdx], idParts[vmNamePartIdx], nil -} diff --git a/provider/v2/azure/scanner/task.go b/provider/v2/azure/scanner/task.go new file mode 100644 index 0000000000..0ee4f39d27 --- /dev/null +++ b/provider/v2/azure/scanner/task.go @@ -0,0 +1,298 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:wrapcheck +package scanner + +import ( + "context" + "fmt" + "reflect" + "strings" + "sync" + + armcompute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + workflowTypes "github.com/openclarity/vmclarity/workflow/types" + + "github.com/openclarity/vmclarity/provider" + "github.com/openclarity/vmclarity/provider/v2/azure/utils" +) + +const ( + EnsureAssetVMInfoTaskName = "EnsureVMInfo" + EnsureSnapshotTaskName = "EnsureSnapshot" + EnsureDiskTaskName = "EnsureDisk" + EnsureScannerVMTaskName = "EnsureScannerVM" + AttachDiskToScannerVMTaskName = "AttachDiskToScannerVM" + + EnsureBlobDeletedTaskName = "EnsureBlobDeleted" + EnsureSnapshotDeletedTaskName = "EnsureSnapshotDeleted" + EnsureTargetDiskDeletedTaskName = "EnsureTargetDiskDeleted" + EnsureScannerVMDeletedTaskName = "EnsureScannerVMDeleted" + EnsureNetworkInterfaceDeletedTaskName = "EnsureNetworkInterfaceDeleted" +) + +type AssetScanState struct { + config *provider.ScanJobConfig + mu *sync.RWMutex + + assetVM armcompute.VirtualMachinesClientGetResponse + scannerVM armcompute.VirtualMachine + snapshot armcompute.Snapshot + disk armcompute.Disk +} + +func (s *AssetScanState) AddAssetVM(assetVM armcompute.VirtualMachinesClientGetResponse) { + s.mu.Lock() + defer s.mu.Unlock() + + s.assetVM = assetVM +} + +func (s *AssetScanState) AddScannerVM(scannerVM armcompute.VirtualMachine) { + s.mu.Lock() + defer s.mu.Unlock() + + s.scannerVM = scannerVM +} + +func (s *AssetScanState) AddSnapshot(snapshot armcompute.Snapshot) { + s.mu.Lock() + defer s.mu.Unlock() + + s.snapshot = snapshot +} + +func (s *AssetScanState) AddDisk(disk armcompute.Disk) { + s.mu.Lock() + defer s.mu.Unlock() + + s.disk = disk +} + +// Ensures that the virtual machine information is available in the workflow's state. +func (s *Scanner) EnsureAssetVMInfo(deps []string) { + s.RunAssetScanTasks = append(s.RunAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureAssetVMInfoTaskName, + Deps: deps, + Fn: func(ctx context.Context, state *AssetScanState) error { + vmInfo, err := state.config.AssetInfo.AsVMInfo() + if err != nil { + return provider.FatalErrorf("unable to get vminfo from asset: %w", err) + } + + resourceGroup, vmName, err := resourceGroupAndNameFromInstanceID(vmInfo.InstanceID) + if err != nil { + return err + } + + assetVM, err := s.VMClient.Get(ctx, resourceGroup, vmName, nil) + if err != nil { + _, err = utils.HandleAzureRequestError(err, "getting asset virtual machine %s", vmName) + return err + } + + state.AddAssetVM(assetVM) + + return nil + }, + }) +} + +// Ensures that a snapshot is created for the asset virtual machine's root volume, +// and that the snapshot is deleted after the workflow has completed. +func (s *Scanner) EnsureSnapshotWithCleanup(deps []string) { + s.RunAssetScanTasks = append(s.RunAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureSnapshotTaskName, + Deps: deps, + Fn: func(ctx context.Context, state *AssetScanState) error { + if reflect.DeepEqual(state.assetVM.VirtualMachine, armcompute.VirtualMachine{}) { + return provider.FatalErrorf("assetVM is not available in the AssetScanState") + } + + snapshot, err := s.ensureSnapshotForVMRootVolume(ctx, state.config, state.assetVM.VirtualMachine) + if err != nil { + return fmt.Errorf("failed to ensure snapshot for vm root volume: %w", err) + } + + state.AddSnapshot(snapshot) + + return nil + }, + }) + + s.RemoveAssetScanTasks = append(s.RemoveAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureSnapshotDeletedTaskName, + Deps: nil, + Fn: func(ctx context.Context, state *AssetScanState) error { + err := s.ensureSnapshotDeleted(ctx, state.config) + if err != nil { + return fmt.Errorf("failed to ensure snapshot deleted: %w", err) + } + + return nil + }, + }) +} + +// Ensures that a managed disk is created from the snapshot, +// and that the disk and the copied blob is deleted after the workflow has completed. +func (s *Scanner) EnsureDiskWithCleanup(deps []string) { + s.RunAssetScanTasks = append(s.RunAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureDiskTaskName, + Deps: deps, + Fn: func(ctx context.Context, state *AssetScanState) error { + var disk armcompute.Disk + var err error + + if reflect.DeepEqual(state.assetVM.VirtualMachine, armcompute.VirtualMachine{}) { + return provider.FatalErrorf("assetVM is not available in the AssetScanState") + } + + if reflect.DeepEqual(state.snapshot, armcompute.Snapshot{}) { + return provider.FatalErrorf("snapshot is not available in the AssetScanState") + } + + if *state.assetVM.Location == s.ScannerLocation { + disk, err = s.ensureManagedDiskFromSnapshot(ctx, state.config, state.snapshot) + if err != nil { + return fmt.Errorf("failed to ensure managed disk created from snapshot: %w", err) + } + } else { + disk, err = s.ensureManagedDiskFromSnapshotInDifferentRegion(ctx, state.config, state.snapshot) + if err != nil { + return fmt.Errorf("failed to ensure managed disk from snapshot in different region: %w", err) + } + } + + state.AddDisk(disk) + + return nil + }, + }) + + s.RemoveAssetScanTasks = append(s.RemoveAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureTargetDiskDeletedTaskName, + Deps: nil, + Fn: func(ctx context.Context, state *AssetScanState) error { + err := s.ensureTargetDiskDeleted(ctx, state.config) + if err != nil { + return fmt.Errorf("failed to ensure asset disk deleted: %w", err) + } + + return nil + }, + }) + + // In case of cross-region snapshot, we need to ensure the copied blob is deleted as well. + s.RemoveAssetScanTasks = append(s.RemoveAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureBlobDeletedTaskName, + Deps: nil, + Fn: func(ctx context.Context, state *AssetScanState) error { + err := s.ensureBlobDeleted(ctx, state.config) + if err != nil { + return fmt.Errorf("failed to ensure snapshot copy blob deleted: %w", err) + } + + return nil + }, + }) +} + +// Ensures that a network interface and the scanner virtual machine is created, +// and that the network interface and the scanner virtual machine are deleted after the workflow has completed. +func (s *Scanner) EnsureScannerVMWithCleanup(deps []string) { + s.RunAssetScanTasks = append(s.RunAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureScannerVMTaskName, + Deps: deps, + Fn: func(ctx context.Context, state *AssetScanState) error { + networkInterface, err := s.ensureNetworkInterface(ctx, state.config) + if err != nil { + return fmt.Errorf("failed to ensure scanner network interface: %w", err) + } + + scannerVM, err := s.ensureScannerVirtualMachine(ctx, state.config, networkInterface) + if err != nil { + return fmt.Errorf("failed to ensure scanner virtual machine: %w", err) + } + + state.AddScannerVM(scannerVM) + + return nil + }, + }) + + s.RemoveAssetScanTasks = append(s.RemoveAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureScannerVMDeletedTaskName, + Deps: nil, + Fn: func(ctx context.Context, state *AssetScanState) error { + err := s.ensureScannerVirtualMachineDeleted(ctx, state.config) + if err != nil { + return fmt.Errorf("failed to ensure scanner virtual machine deleted: %w", err) + } + return nil + }, + }) + + s.RemoveAssetScanTasks = append(s.RemoveAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: EnsureNetworkInterfaceDeletedTaskName, + Deps: []string{EnsureScannerVMDeletedTaskName}, + Fn: func(ctx context.Context, state *AssetScanState) error { + err := s.ensureNetworkInterfaceDeleted(ctx, state.config) + if err != nil { + return fmt.Errorf("failed to ensure network interface deleted: %w", err) + } + + return nil + }, + }) +} + +// Ensures that the asset disk is attached to the scanner virtual machine. +func (s *Scanner) EnsureAttachDiskToScannerVM(deps []string) { + s.RunAssetScanTasks = append(s.RunAssetScanTasks, &workflowTypes.Task[*AssetScanState]{ + Name: AttachDiskToScannerVMTaskName, + Deps: deps, + Fn: func(ctx context.Context, state *AssetScanState) error { + if reflect.DeepEqual(state.assetVM.VirtualMachine, armcompute.VirtualMachine{}) { + return provider.FatalErrorf("assetVM is not available in the AssetScanState") + } + + if reflect.DeepEqual(state.disk, armcompute.Disk{}) { + return provider.FatalErrorf("disk is not available in the AssetScanState") + } + + err := s.ensureDiskAttachedToScannerVM(ctx, state.scannerVM, state.disk) + if err != nil { + return fmt.Errorf("failed to ensure asset disk is attached to virtual machine: %w", err) + } + + return nil + }, + }) +} + +// Example Instance ID: +// +// /subscriptions/ecad88af-09d5-4725-8d80-906e51fddf02/resourceGroups/vmclarity-sambetts-dev/providers/Microsoft.Compute/virtualMachines/vmclarity-server +// +// Will return "vmclarity-sambetts-dev" and "vmclarity-server". +func resourceGroupAndNameFromInstanceID(instanceID string) (string, string, error) { + idParts := strings.Split(instanceID, "/") + if len(idParts) != instanceIDPartsLength { + return "", "", provider.FatalErrorf("asset instance id in unexpected format got: %s", idParts) + } + return idParts[resourceGroupPartIdx], idParts[vmNamePartIdx], nil +}