Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: calculate monthly cost for bucket #650

Merged
merged 7 commits into from
Mar 24, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ require (
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.12.0 // indirect
cloud.google.com/go/monitoring v1.13.0 // indirect
cloud.google.com/go/storage v1.30.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE=
cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
cloud.google.com/go/monitoring v1.13.0 h1:2qsrgXGVoRXpP7otZ14eE1I568zAa92sJSDPyOJvwjM=
cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw=
cloud.google.com/go/storage v1.30.0 h1:g1yrbxAWOrvg/594228pETWkOi00MLTrOWfh56veU5o=
cloud.google.com/go/storage v1.30.0/go.mod h1:xAVretHSROm1BQX4IIsoVgJqw0LqOyX+I/O2GzRAzdE=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.1 h1:gVXuXcWd1i4C2Ruxe321aU+IKGaStvGB/S90PUPB/W8=
Expand Down
72 changes: 72 additions & 0 deletions providers/gcp/storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,75 @@ package storage

import (
"context"
"errors"
"fmt"
"strings"
"time"

monitoring "cloud.google.com/go/monitoring/apiv3/v2"
monitoringpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
"cloud.google.com/go/storage"
"github.com/golang/protobuf/ptypes/duration"
"github.com/golang/protobuf/ptypes/timestamp"
log "github.com/sirupsen/logrus"
"github.com/tailwarden/komiser/models"
"github.com/tailwarden/komiser/providers"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)

// GetBucketSize returns the size of a bucket in bytes
func GetBucketSize(ctx context.Context, client providers.ProviderClient, bucketName string) (int64, error) {
monitoringClient, err := monitoring.NewMetricClient(ctx, option.WithCredentials(client.GCPClient.Credentials))
if err != nil {
log.WithError(err).Debug("failed to create monitoring client")
return 0, err
}

beginningOfMonth := time.Date(time.Now().Year(), time.Now().Month(), 1, 0, 0, 0, 0, time.UTC).Unix()

req := &monitoringpb.ListTimeSeriesRequest{
Name: "projects/" + client.GCPClient.Credentials.ProjectID,
Filter: "metric.type=\"storage.googleapis.com/storage/total_bytes\" resource.type=\"gcs_bucket\" resource.label.bucket_name=\"" + bucketName + "\"",
Interval: &monitoringpb.TimeInterval{
StartTime: &timestamp.Timestamp{Seconds: beginningOfMonth},
EndTime: &timestamp.Timestamp{Seconds: time.Now().Unix()},
},
Aggregation: &monitoringpb.Aggregation{
AlignmentPeriod: &duration.Duration{Seconds: 60},
PerSeriesAligner: monitoringpb.Aggregation_ALIGN_SUM,
},
}

res := monitoringClient.ListTimeSeries(ctx, req)

for {
series, err := res.Next()
if err == iterator.Done {
break
}
if err != nil {
log.WithError(err).Debug("failed to list time series")
return 0, err
}

for _, point := range series.Points {
return int64(point.Value.GetDoubleValue()), nil
}
}

return 0, errors.New("no data found")
}

func getPricingForGCPBuckets() map[string]float64 {
return map[string]float64{
"STANDARD": 0.20,
"NEARLINE": 0.10,
"COLDLINE": 0.01,
"ARCHIVE": 0.005,
}
}

func Buckets(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) {
resources := make([]models.Resource, 0)

Expand Down Expand Up @@ -43,6 +100,20 @@ func Buckets(ctx context.Context, client providers.ProviderClient) ([]models.Res
}
}

monthlyCost := 0.0

bucketSize, err := GetBucketSize(ctx, client, bucket.Name)
if err != nil {
log.WithError(err).WithFields(log.Fields{
"bucket": bucket.Name,
}).Debug("failed to get bucket size, skipping cost calculation")
}

if bucketSize > 0 {
bucketSizeInGB := float64(bucketSize) / 1024 / 1024 / 1024
monthlyCost = float64(bucketSizeInGB) * getPricingForGCPBuckets()[strings.ToUpper(bucket.StorageClass)]
}

resources = append(resources, models.Resource{
Provider: "GCP",
Account: client.Name,
Expand All @@ -51,6 +122,7 @@ func Buckets(ctx context.Context, client providers.ProviderClient) ([]models.Res
ResourceId: bucket.Name,
Region: strings.ToLower(bucket.Location),
Tags: tags,
Cost: monthlyCost,
FetchedAt: time.Now(),
Link: fmt.Sprintf("https://console.cloud.google.com/storage/browser/%s?project=%s", bucket.Name, client.GCPClient.Credentials.ProjectID),
})
Expand Down