cmd/scollector: Added Azure Enterprise Agreement billing collector

cmd/scollector/collectors/azureeabilling.go

@@ -0,0 +1,194 @@
+package collectors
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mhenderson-so/azure-ea-billing"
+
+	"bosun.org/metadata"
+	"bosun.org/opentsdb"
+)
+
+var azBillConf = azureEABillingConfig{}
+
+const (
+	hoursInDay = 24
+	usageDesc  = "Usage of Azure service. Category is concatenated meter details. Resource is concatenated resource group and resource name."
+	costDesc   = "Cost of Azure service. Category is concatenated meter details. Resource is concatenated resource group and resource name."
+	priceDesc  = "Azure price sheet data for Enterprise Agreement services"
+)
+
+func AzureEABilling(ea uint32, key string, logBilling bool) error {
+	if ea > 0 && key != "" {
+		azBillConf = azureEABillingConfig{
+			AZEABillingConfig: azureeabilling.Config{
+				EA:     ea,
+				APIKey: key,
+			},
+			LogBillingDetails: logBilling,
+		}
+
+		collectors = append(collectors, &IntervalCollector{
+			F:        c_azureeabilling,
+			Interval: 1 * time.Hour,
+		})
+	}
+
+	return nil
+}
+
+func c_azureeabilling() (opentsdb.MultiDataPoint, error) {
+	var md opentsdb.MultiDataPoint
+
+	//Get the list of available bills from the portal
+	reports, err := azBillConf.AZEABillingConfig.GetUsageReports()
+	if err != nil {
+		return nil, err
+	}
+
+	//Process the report list
+	if err = processAzureEAReports(reports, &md); err != nil {
+		return nil, err
+	}
+
+	return md, nil
+
+}
+
+// processAzureEAReports will go through the monthly reports provided and pull out the ones that we're going to process
+func processAzureEAReports(reports *azureeabilling.UsageReports, md *opentsdb.MultiDataPoint) error {
+	baseTime := time.Now()
+	thisMonth := baseTime.Format("2006-01")
+	lastMonth := time.Date(baseTime.Year(), baseTime.Month()-1, 1, 0, 0, 0, 0, time.UTC).Format("2006-01")
+	for _, r := range reports.AvailableMonths {
+		//There's potentially a lot of reports. We only want to process this months + last months report
+		if !(thisMonth == r.Month || lastMonth == r.Month) {
+			return nil
+		}
+
+		csv := azBillConf.AZEABillingConfig.GetMonthReportsCSV(r, azureeabilling.DownloadForStructs)
+		structs, err := csv.ConvertToStructs()
+
+		if err != nil {
+			return err
+		}
+		for _, p := range structs.PriceSheetReport {
+			err := processAzureEAPriceSheetRow(p, md)
+			if err != nil {
+				return err
+			}
+		}
+		for _, d := range structs.DetailReport {
+			err := processAzureEADetailRow(d, md)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// processAzureEAPriceSheetRow will take the price sheet info and log it, so we can track price changes over time
+func processAzureEAPriceSheetRow(p *azureeabilling.PriceSheetRow, md *opentsdb.MultiDataPoint) error {
+	fullProdName := fmt.Sprintf("%s-%s", p.Service, p.UnitOfMeasure)
+	priceString := convertAzurePriceToString(p.UnitPrice)
+	tags := opentsdb.TagSet{
+		"partnumber": p.PartNumber,
+		"service":    fullProdName,
+	}
+	Add(md, "azure.ea.pricesheet", priceString, tags, metadata.Gauge, metadata.Count, priceDesc)
+	return nil
+}
+
+// processAzureEADetailRow will take the actual usage data for the provided month
+func processAzureEADetailRow(p *azureeabilling.DetailRow, md *opentsdb.MultiDataPoint) error {
+	//Don't process todays records as they are subject to change
+	nowYear, nowMonth, nowDay := time.Now().Date()
+	recordMonth := int(nowMonth)
+	if nowYear == p.Year && recordMonth == p.Month && nowDay == p.Day {
+		return nil
+	}
+
+	resourcePaths := strings.Split(strings.ToLower(p.InstanceID), "/")
+	var resourceString string
+
+	if len(resourcePaths) < 8 {
+		resourceString = strings.ToLower(p.InstanceID)
+	} else {
+		resourceIDs := resourcePaths[8:]
+		resourceString = strings.Join(resourceIDs, "-")
+	}
+
+	if p.ResourceGroup != "" {
+		resourceString = fmt.Sprintf("%s-%s", strings.ToLower(p.ResourceGroup), resourceString)
+	}
+
+	tags := opentsdb.TagSet{
+		"category":    p.MeterCategory,
+		"subcategory": fmt.Sprintf("%s-%s", strings.ToLower(p.MeterSubCategory), strings.ToLower(p.MeterName)),
+	}
+
+	resourceString, err := opentsdb.Clean(resourceString)
+	if err != nil && resourceString != "" {
+		tags["resource"] = resourceString
+	}
+
+	//Only log billing details if they are enabled in the config
+	if azBillConf.LogBillingDetails {
+		if p.CostCenter != "" {
+			tags["costcenter"] = strings.ToLower(p.CostCenter)
+		}
+		cleanAccountName, _ := opentsdb.Clean(p.AccountName)
+		tags["accountname"] = strings.ToLower(cleanAccountName)
+		tags["subscription"] = strings.ToLower(p.SubscriptionName)
+	}
+
+	recordDate := time.Date(p.Year, time.Month(p.Month), p.Day, 0, 0, 0, 0, time.UTC)
+
+	//Because we need to log this hourly and we only have daily data, divide the daily cost into hourly costs
+	qtyPerHour := p.ConsumedQuantity / hoursInDay
+
+	//ExtendedCost is stored only in a string, because it's a variable number of decimal places. Which means we can't reliably store it in an int, and storing in a float reduces precision.
+	//This way we're choosing ourselves to drop the precision, which adds up to around 10-20c under initial testing.
+	costPerDay, err := strconv.ParseFloat(p.ExtendedCostRaw, 64)
+	if err != nil {
+		return err
+	}
+	costPerHour := costPerDay / hoursInDay
+
+	//Get 24 records for 24 hours in a day
+	for i := 0; i < hoursInDay; i++ {
+		recordTime := recordDate.Add(time.Duration(i) * time.Hour)
+		AddTS(md, "azure.ea.usage", recordTime.Unix(), qtyPerHour, tags, metadata.Gauge, metadata.Count, usageDesc)
+		AddTS(md, "azure.ea.cost", recordTime.Unix(), costPerHour, tags, metadata.Gauge, metadata.Count, costDesc)
+	}
+
+	return nil
+}
+
+//The cost is stored in cents, and we want to translate the cent cost into dollars and cents, but in a string
+//which will not lose precision and is close enough for government work.
+func convertAzurePriceToString(costInCents int) string {
+	priceString := strconv.Itoa(costInCents)
+	priceLen := len(priceString)
+	if priceLen == 1 {
+		priceString = fmt.Sprintf("0.0%s", priceString)
+	}
+	if priceLen == 2 {
+		priceString = fmt.Sprintf("0.%s", priceString)
+	}
+	if priceLen >= 3 {
+		priceString = fmt.Sprintf("%s.%s", priceString[0:priceLen-2], priceString[priceLen-2:])
+	}
+
+	return priceString
+}
+
+type azureEABillingConfig struct {
+	LogBillingDetails bool
+	AZEABillingConfig azureeabilling.Config
+}

cmd/scollector/conf/conf.go

@@ -61,6 +61,7 @@ type Conf struct {
 	ICMP           []ICMP
 	Vsphere        []Vsphere
 	AWS            []AWS
+	AzureEA        []AzureEA
 	Process        []ProcessParams
 	SystemdService []ServiceParams
 	ProcessDotNet  []ProcessDotNet
@@ -148,6 +149,12 @@ type AWS struct {
 	BillingPurgeDays         int
 }
 
+type AzureEA struct {
+	EANumber          uint32
+	APIKey            string
+	LogBillingDetails bool
+}
+
 type SNMP struct {
 	Community string
 	Host      string

cmd/scollector/doc.go

@@ -235,6 +235,28 @@ data sent to OpenTSDB or Bosun.
 	  BillingBucketPath = "reports"
 	  BillingPurgeDays = 2
 
+
+AzureEA (array of table, keys are EANumber, APIKey and LogBillingDetails): Azure Enterprise
+Agreements to poll for billing information.
+
+EANumber is your Enterprise Agreement number. You can find this in your Enterprise Agreement portal.
+
+APIKey is the API key as provided by the Azure EA Portal. To generate your API key for this collector,
+you will need to log into your Azure Enterprise Agreement portal (ea.azure.com), click the
+"Download Usage" link, then choose "API Key" on the download page. You can then generate your API
+key there. Keys are valid 6 months, so you will require some maintenance of this collector twice a year.
+
+LogBillingDetails tells scollector to add the following tags to your metrics:
+   - costcenter
+	 - accountname
+	 - subscription
+If you are a heavy Azure EA user, then these additional tags may be useful for breaking down costs.
+
+	[[AzureEA]]
+	  EANumber = "123456"
+	  APIKey = "joiIiwiaXNzIjoiZWEubWljcm9zb2Z0YXp1cmUuY29tIiwiYXVkIjoiY2xpZW50LmVhLm1"
+	  LogBillingDetails = false
+
 Process: processes to monitor.
 
 ProcessDotNet: .NET processes to monitor on Windows.

cmd/scollector/main.go

@@ -150,6 +150,9 @@ func main() {
 	for _, a := range conf.AWS {
 		check(collectors.AWS(a.AccessKey, a.SecretKey, a.Region, a.BillingProductCodesRegex, a.BillingBucketName, a.BillingBucketPath, a.BillingPurgeDays))
 	}
+	for _, ea := range conf.AzureEA {
+		check(collectors.AzureEABilling(ea.EANumber, ea.APIKey, ea.LogBillingDetails))
+	}
 	for _, v := range conf.Vsphere {
 		check(collectors.Vsphere(v.User, v.Password, v.Host))
 	}