Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cleanup job in case of a failure #11

Merged
merged 2 commits into from
Nov 23, 2022
Merged

cleanup job in case of a failure #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
09483ee
cleanup job in case of a failure
Nov 22, 2022
251b8a6
review
Nov 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion runtime_scan/pkg/provider/aws/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func (c *Client) Discover(ctx context.Context, scanScope types.ScanScope) ([]typ
return ret, nil
}

func (c *Client) LaunchInstance(ctx context.Context, snapshot types.Snapshot) (types.Instance, error) {
func (c *Client) RunScanningJob(ctx context.Context, snapshot types.Snapshot) (types.Instance, error) {
out, err := c.ec2Client.RunInstances(ctx, &ec2.RunInstancesInput{
MaxCount: utils.Int32Ptr(1),
MinCount: utils.Int32Ptr(1),
Expand Down
2 changes: 1 addition & 1 deletion runtime_scan/pkg/provider/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,5 @@ type Client interface {
// Discover - list VM instances in the account according to the scan scope.
Discover(ctx context.Context, scanScope types.ScanScope) ([]types.Instance, error)
// RunScanningJob - run a scanning job
LaunchInstance(ctx context.Context, snapshot types.Snapshot) (types.Instance, error)
RunScanningJob(ctx context.Context, snapshot types.Snapshot) (types.Instance, error)
}
61 changes: 38 additions & 23 deletions runtime_scan/pkg/scanner/job_managment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,48 +149,57 @@ func (s *Scanner) waitForResult(data *scanData, ks chan bool) {
}
}

// TODO need to clean all the created resources in case of a failure.
func (s *Scanner) runJob(ctx context.Context, data *scanData) (types.Job, error) {
instanceToScan := data.instance
var launchInstance types.Instance
var launchSnapshot types.Snapshot
var cpySnapshot types.Snapshot
var snapshot types.Snapshot
var job types.Job
var err error

instanceToScan := data.instance

// cleanup in case of an error
defer func() {
if err != nil {
s.deleteJob(ctx, &job)
}
}()

volume, err := instanceToScan.GetRootVolume(ctx)
FrimIdan marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return types.Job{}, fmt.Errorf("failed to get root volume of an instance %v: %v", instanceToScan.GetID(), err)
}

snapshot, err := volume.TakeSnapshot(ctx)
snapshot, err = volume.TakeSnapshot(ctx)
if err != nil {
return types.Job{}, fmt.Errorf("failed to take snapshot of a volume: %v", err)
}
if err := snapshot.WaitForReady(ctx); err != nil {
return types.Job{}, fmt.Errorf("failed to wait for snapshot %v ready: %v", snapshot.GetID(), err)
}
job.SrcSnapshot = snapshot
launchSnapshot = snapshot
if err = snapshot.WaitForReady(ctx); err != nil {
return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready. snapshotID=%v: %v", snapshot.GetID(), err)
}

if s.region != snapshot.GetRegion() {
cpySnapshot, err = snapshot.Copy(ctx, s.region)
if err != nil {
return types.Job{}, fmt.Errorf("failed to copy snapshot %v: %v", snapshot.GetID(), err)
}

if err := cpySnapshot.WaitForReady(ctx); err != nil {
return types.Job{}, fmt.Errorf("failed wait for snapshot %v ready: %v", cpySnapshot.GetID(), err)
return types.Job{}, fmt.Errorf("failed to copy snapshot. snapshotID=%v: %v", snapshot.GetID(), err)
}
job.DstSnapshot = cpySnapshot
launchSnapshot = cpySnapshot
if err = cpySnapshot.WaitForReady(ctx); err != nil {
return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready. snapshotID=%v: %v", cpySnapshot.GetID(), err)
}
}

i, err := s.providerClient.LaunchInstance(ctx, launchSnapshot)
launchInstance, err = s.providerClient.RunScanningJob(ctx, launchSnapshot)
if err != nil {
return types.Job{}, fmt.Errorf("failed to launch a new instance: %v", err)
}
job.Instance = launchInstance

return types.Job{
Instance: i,
SrcSnapshot: snapshot,
DstSnapshot: cpySnapshot,
}, nil
return job, nil
}

func (s *Scanner) deleteJobIfNeeded(ctx context.Context, job *types.Job, isSuccessfulJob, isCompletedJob bool) {
Expand All @@ -217,13 +226,19 @@ func (s *Scanner) deleteJobIfNeeded(ctx context.Context, job *types.Job, isSucce
}

func (s *Scanner) deleteJob(ctx context.Context, job *types.Job) {
if err := job.Instance.Delete(ctx); err != nil {
log.Errorf("failed to delete instance: %v", err)
if job.Instance != nil {
if err := job.Instance.Delete(ctx); err != nil {
log.Errorf("Failed to delete instance. instanceID=%v: %v", job.Instance.GetID(), err)
}
}
if err := job.SrcSnapshot.Delete(ctx); err != nil {
log.Errorf("failed to delete source snapshot: %v", err)
if job.SrcSnapshot != nil {
if err := job.SrcSnapshot.Delete(ctx); err != nil {
log.Errorf("Failed to delete source snapshot. snapshotID=%v: %v", job.SrcSnapshot.GetID(), err)
}
}
if err := job.DstSnapshot.Delete(ctx); err != nil {
log.Errorf("failed to delete dest snapshot: %v", err)
if job.DstSnapshot != nil {
if err := job.DstSnapshot.Delete(ctx); err != nil {
log.Errorf("Failed to delete destination snapshot. snapshotID=%v: %v", job.DstSnapshot.GetID(), err)
}
}
}