Memory fixes

pkg/calculator/calculator.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/cnkei/gospline"
@@ -15,10 +16,8 @@ import (
 type parameters struct {
 	grid           float64
 	pue            float64
-	powerCPU       []data.Wattage
-	powerRAM       []data.Wattage
 	metric         *v1.Metric
-	vCPU           float64
+	factors        *data.Instance
 	embodiedFactor float64
 }
 
@@ -30,7 +29,7 @@ func operationalEmissions(ctx context.Context, interval time.Duration, p *parame
 	case v1.CPU.String():
 		return cpu(ctx, interval, p)
 	case v1.Memory.String():
-		return memory(ctx, p)
+		return memory(ctx, interval, p)
 	case v1.Storage.String():
 		return errors.New("error storage is not yet being calculated")
 	case v1.Network.String():
@@ -49,7 +48,8 @@ func operationalEmissions(ctx context.Context, interval time.Duration, p *parame
 func cpu(ctx context.Context, interval time.Duration, p *parameters) error {
 	logger := log.FromContext(ctx)
 
-	vCPU := p.vCPU
+	// TODO: remove casting once the type is changed in the factors data
+	vCPU := float64(p.factors.VCPU)
 	// vCPU are virtual CPUs that are mapped to physical cores (a core is a physical
 	// component to the CPU the VM is running on). If vCPU from the dataset (p.vCPU)
 	// is not found, get the number of vCPUs from the metric collected from the query
@@ -70,7 +70,7 @@ func cpu(ctx context.Context, interval time.Duration, p *parameters) error {
 	// energy is the CPU energy consumption in kilowatts.
 	// If pkgWatt values exist from the dataset, then use cubic spline interpolation
 	// to calculate the wattage based on utilization.
-	usage, err := cubicSplineInterpolation(p.powerCPU, p.metric.Usage)
+	usage, err := cubicSplineInterpolation(p.factors.PkgWatt, p.metric.Usage)
 	if err != nil {
 		return err
 	}
@@ -92,19 +92,28 @@ func cpu(ctx context.Context, interval time.Duration, p *parameters) error {
 // memory is calculated based on the TEADs pkgRAM calculations over
 // various memory stress loads. Using the memory usage from the instance
 // we can get the estimated Power consumption of the instance
-func memory(ctx context.Context, p *parameters) error {
+func memory(ctx context.Context, interval time.Duration, p *parameters) error {
 	logger := log.FromContext(ctx)
 	var err error
 
-	if p.powerRAM == nil {
+	if reflect.DeepEqual(p.factors.RAMWatt, emptyWattage) {
 		return fmt.Errorf("not calculating memory - RAM wattage data not found")
 	}
 
-	p.metric.Energy, err = cubicSplineInterpolation(p.powerRAM, p.metric.Usage)
+	if p.metric.Usage == 0 {
+		return fmt.Errorf("no memory usage found")
+	}
+
+	p.metric.Energy, err = cubicSplineInterpolation(p.factors.RAMWatt, p.metric.Usage)
 	if err != nil {
 		return err
 	}
 
+	// RAMWatt is the energy consumption for a single GB of memory, so multiply
+	// the energy usage by the total instance memoryHours to get the energy consumption
+	memoryHours := (interval.Minutes() / float64(60)) * p.factors.MemoryGB
+	p.metric.Energy *= memoryHours
+
 	p.metric.Emissions = v1.NewResourceEmission(
 		p.metric.Energy*p.pue*p.grid,
 		v1.GCO2eq,

pkg/calculator/calculator_test.go

@@ -20,43 +20,46 @@ func params() *parameters {
 		pue:    1.2,
 		metric: m,
 		// using t3.micro AWS instance as default
-		powerCPU: []data.Wattage{
-			{
-				Percentage: 0,
-				Wattage:    1.21,
-			},
-			{
-				Percentage: 10,
-				Wattage:    3.05,
-			},
-			{
-				Percentage: 50,
-				Wattage:    7.16,
-			},
-			{
-				Percentage: 100,
-				Wattage:    9.96,
-			},
-		},
-		powerRAM: []data.Wattage{
-			{
-				Percentage: 0,
-				Wattage:    0.15,
-			},
-			{
-				Percentage: 10,
-				Wattage:    0.24,
-			},
-			{
-				Percentage: 50,
-				Wattage:    0.62,
+		factors: &data.Instance{
+			PkgWatt: []data.Wattage{
+				{
+					Percentage: 0,
+					Wattage:    1.21,
+				},
+				{
+					Percentage: 10,
+					Wattage:    3.05,
+				},
+				{
+					Percentage: 50,
+					Wattage:    7.16,
+				},
+				{
+					Percentage: 100,
+					Wattage:    9.96,
+				},
 			},
-			{
-				Percentage: 100,
-				Wattage:    1.00,
+			RAMWatt: []data.Wattage{
+				{
+					Percentage: 0,
+					Wattage:    0.15,
+				},
+				{
+					Percentage: 10,
+					Wattage:    0.24,
+				},
+				{
+					Percentage: 50,
+					Wattage:    0.62,
+				},
+				{
+					Percentage: 100,
+					Wattage:    1.00,
+				},
 			},
+			VCPU:     2.0,
+			MemoryGB: 1,
 		},
-		vCPU:           2,
 		embodiedFactor: 1000,
 	}
 }
@@ -89,7 +92,7 @@ func TestCalculateCPU(t *testing.T) {
 		func() *testcase {
 			// vCPUs not set in params, but set in metric
 			p := params()
-			p.vCPU = 0
+			p.factors.VCPU = 0
 			p.metric.UnitAmount = 2
 			p.metric.Unit = v1.VCPU
 			return &testcase{
@@ -106,7 +109,7 @@ func TestCalculateCPU(t *testing.T) {
 		func() *testcase {
 			// vCPUs not set
 			p := params()
-			p.vCPU = 0
+			p.factors.VCPU = 0
 			return &testcase{
 				name:     "vCPU not set",
 				interval: 5 * time.Minute,
@@ -155,7 +158,7 @@ func TestCalculateCPU(t *testing.T) {
 		func() *testcase {
 			// calculate with 4 vCPUs
 			p := params()
-			p.vCPU = 4
+			p.factors.VCPU = 4
 			return &testcase{
 				name:     "4 vCPU",
 				interval: 5 * time.Minute,
@@ -320,6 +323,7 @@ func TestCalculateMemory(t *testing.T) {
 	type testcase struct {
 		name      string
 		params    *parameters
+		interval  time.Duration // this is nanoseconds
 		emissions float64
 		energy    float64
 		hasErr    bool
@@ -331,14 +335,15 @@ func TestCalculateMemory(t *testing.T) {
 			return &testcase{
 				name:      "default t3.micro at 27%",
 				params:    params(),
-				energy:    0.00040240120731707316,
-				emissions: 0.0033801701414634144,
+				interval:  5 * time.Minute,
+				energy:    3.353343394308943e-05,
+				emissions: 0.0002816808451219512,
 			}
 		}(),
 		func() *testcase {
 			// fail: powerRAM wattage not set
 			p := params()
-			p.powerRAM = []data.Wattage{}
+			p.factors.RAMWatt = []data.Wattage{}
 			return &testcase{
 				name:      "fail: wattage RAM data not set",
 				params:    p,
@@ -350,7 +355,7 @@ func TestCalculateMemory(t *testing.T) {
 		}(),
 	} {
 		t.Run(test.name, func(t *testing.T) {
-			err := memory(context.TODO(), test.params)
+			err := memory(context.TODO(), test.interval, test.params)
 			actualEmissions := test.params.metric.Emissions.Value
 			actualEnergy := test.params.metric.Energy
 

pkg/calculator/handler.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
+	"reflect"
 
 	"gopkg.in/yaml.v2"
 
@@ -16,14 +17,17 @@ import (
 	data "github.com/re-cinq/emissions-data/pkg/types/v2"
 )
 
-var awsInstances map[string]data.Instance
+var instanceData map[string]data.Instance
 
 // AWS, GCP and Azure have increased their server lifespan to 6 years (2024)
 // https://sustainability.aboutamazon.com/products-services/the-cloud?energyType=true
 // https://www.theregister.com/2024/01/31/alphabet_q4_2023/
 // https://www.theregister.com/2022/08/02/microsoft_server_life_extension/
 const serverLifespan = 6
 
+//nolint:govet // don't need to write the struct fields each time
+var emptyWattage = []data.Wattage{{0, 0}, {10, 0}, {50, 0}, {100, 0}}
+
 // CalculatorHandler is used to handle events when metrics have been collected
 type CalculatorHandler struct {
 	Bus    *bus.Bus
@@ -41,9 +45,11 @@ func NewHandler(ctx context.Context, b *bus.Bus) *CalculatorHandler {
 		return nil
 	}
 
-	awsInstances, err = getProviderEC2EmissionFactors(v1.AWS)
-	if err != nil {
-		logger.Error("unable to get v2 Emission Factors, falling back to v1", "error", err)
+	for provider := range config.AppConfig().Providers {
+		err = getProviderEmissionFactors(provider)
+		if err != nil {
+			logger.Error("unable to get v2 Emission Factors", "error", err, "provider", provider)
+		}
 	}
 
 	return &CalculatorHandler{
@@ -119,13 +125,15 @@ func (c *CalculatorHandler) handleEvent(e *bus.Event) {
 		pue:  factor.AveragePUE,
 	}
 
-	if d, ok := awsInstances[instance.Kind]; ok {
-		params.powerCPU = d.PkgWatt
-		params.powerRAM = d.RAMWatt
-		params.vCPU = float64(d.VCPU)
-		params.embodiedFactor = d.EmbodiedHourlyGCO2e
-	} else {
-		params.powerCPU = []data.Wattage{
+	if d, ok := instanceData[instance.Kind]; ok {
+		params.factors = &d
+	}
+
+	// fallback to use spec power min and max watt values.
+	// this is less accurate and a place holder until a
+	// different solution is implemented.
+	if reflect.DeepEqual(params.factors.PkgWatt, emptyWattage) {
+		params.factors.PkgWatt = []data.Wattage{
 			{
 				Percentage: 0,
 				Wattage:    specs.MinWatts,
@@ -135,6 +143,9 @@ func (c *CalculatorHandler) handleEvent(e *bus.Event) {
 				Wattage:    specs.MaxWatts,
 			},
 		}
+	}
+
+	if params.embodiedFactor == 0 {
 		params.embodiedFactor = hourlyEmbodiedEmissions(&specs)
 	}
 
@@ -186,21 +197,21 @@ func hourlyEmbodiedEmissions(e *factors.Embodied) float64 {
 		(e.VCPU / e.TotalVCPU)
 }
 
-func getProviderEC2EmissionFactors(provider v1.Provider) (map[string]data.Instance, error) {
+func getProviderEmissionFactors(provider v1.Provider) error {
 	url := "https://raw.githubusercontent.com/re-cinq/emissions-data/main/data/v2/%s-instances.yaml"
 	u := fmt.Sprintf(url, provider)
 
 	r, err := http.Get(u)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer r.Body.Close()
 
 	d := yaml.NewDecoder(r.Body)
-	err = d.Decode(&awsInstances)
+	err = d.Decode(&instanceData)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	return awsInstances, nil
+	return nil
 }

pkg/providers/gcp/instance.go

@@ -100,7 +100,7 @@ func (c *Client) memoryMetrics(ctx context.Context, project, query string) error
 		m.Unit = v1.GB
 		m.ResourceType = v1.Memory
 		// convert Bytes to GB
-		m.Usage = float64(resp.GetPointData()[0].GetValues()[0].GetInt64Value()) / 1024 / 1024 / 1024
+		m.UnitAmount = float64(resp.GetPointData()[0].GetValues()[0].GetInt64Value()) / 1024 / 1024 / 1024
 		m.Labels = v1.Labels{
 			"id":           instanceID,
 			"name":         instanceName,