From c3cec9883fc6aa7ad548a1a56d5a448475c80cdb Mon Sep 17 00:00:00 2001 From: Landan Cheruka Date: Mon, 28 Sep 2020 23:12:29 -0400 Subject: [PATCH] -Added azure fingerprinting support --- client/fingerprint/env_azure.go | 213 ++++++++++++++++++++++++++ client/fingerprint/env_azure_test.go | 219 +++++++++++++++++++++++++++ client/fingerprint/fingerprint.go | 5 +- 3 files changed, 435 insertions(+), 2 deletions(-) create mode 100644 client/fingerprint/env_azure.go create mode 100644 client/fingerprint/env_azure_test.go diff --git a/client/fingerprint/env_azure.go b/client/fingerprint/env_azure.go new file mode 100644 index 000000000000..955d8dc1bc23 --- /dev/null +++ b/client/fingerprint/env_azure.go @@ -0,0 +1,213 @@ +package fingerprint + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" + + cleanhttp "github.com/hashicorp/go-cleanhttp" + log "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/nomad/helper/useragent" + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + // AzureMetadataURL is where the Azure metadata server normally resides. We hardcode the + // "instance" path as well since it's the only one we access here. + AzureMetadataURL = "http://169.254.169.254/metadata/instance/" + + // AzureMetadataAPIVersion is the version used when contacting the Azure metadata + // services. + AzureMetadataAPIVersion = "2019-06-04" + + // AzureMetadataTimeout is the timeout used when contacting the Azure metadata + // services. + AzureMetadataTimeout = 2 * time.Second +) + +type AzureMetadataTag struct { + Name string + Value string +} + +type AzureMetadataPair struct { + path string + unique bool +} + +// EnvAzureFingerprint is used to fingerprint Azure metadata +type EnvAzureFingerprint struct { + StaticFingerprinter + client *http.Client + logger log.Logger + metadataURL string +} + +// NewEnvAzureFingerprint is used to create a fingerprint from Azure metadata +func NewEnvAzureFingerprint(logger log.Logger) Fingerprint { + // Read the internal metadata URL from the environment, allowing test files to + // provide their own + metadataURL := os.Getenv("AZURE_ENV_URL") + if metadataURL == "" { + metadataURL = AzureMetadataURL + } + + // assume 2 seconds is enough time for inside Azure network + client := &http.Client{ + Timeout: AzureMetadataTimeout, + Transport: cleanhttp.DefaultTransport(), + } + + return &EnvAzureFingerprint{ + client: client, + logger: logger.Named("env_azure"), + metadataURL: metadataURL, + } +} + +func (f *EnvAzureFingerprint) Get(attribute string, format string) (string, error) { + reqURL := f.metadataURL + attribute + fmt.Sprintf("?api-version=%s&format=%s", AzureMetadataAPIVersion, format) + parsedURL, err := url.Parse(reqURL) + if err != nil { + return "", err + } + + req := &http.Request{ + Method: "GET", + URL: parsedURL, + Header: http.Header{ + "Metadata": []string{"true"}, + "User-Agent": []string{useragent.String()}, + }, + } + + res, err := f.client.Do(req) + if err != nil { + f.logger.Debug("could not read value for attribute", "attribute", attribute, "error", err) + return "", err + } else if res.StatusCode != http.StatusOK { + f.logger.Debug("could not read value for attribute", "attribute", attribute, "resp_code", res.StatusCode) + return "", err + } + + resp, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + f.logger.Error("error reading response body for Azure attribute", "attribute", attribute, "error", err) + return "", err + } + + if res.StatusCode >= 400 { + return "", ReqError{res.StatusCode} + } + + return string(resp), nil +} + +func checkAzureError(err error, logger log.Logger, desc string) error { + // If it's a URL error, assume we're not actually in an Azure environment. + // To the outer layers, this isn't an error so return nil. + if _, ok := err.(*url.Error); ok { + logger.Debug("error querying Azure attribute; skipping", "attribute", desc) + return nil + } + // Otherwise pass the error through. + return err +} + +func (f *EnvAzureFingerprint) Fingerprint(request *FingerprintRequest, response *FingerprintResponse) error { + cfg := request.Config + + // Check if we should tighten the timeout + if cfg.ReadBoolDefault(TightenNetworkTimeoutsConfig, false) { + f.client.Timeout = 1 * time.Millisecond + } + + if !f.isAzure() { + return nil + } + + // Keys and whether they should be namespaced as unique. Any key whose value + // uniquely identifies a node, such as ip, should be marked as unique. When + // marked as unique, the key isn't included in the computed node class. + keys := map[string]AzureMetadataPair{ + "id": {unique: true, path: "compute/vmId"}, + "hostname": {unique: true, path: "compute/name"}, + "location": {unique: false, path: "compute/location"}, + "resource-group": {unique: false, path: "compute/resourceGroupName"}, + "scale-set": {unique: false, path: "compute/vmScaleSetName"}, + "vm-size": {unique: false, path: "compute/vmSize"}, + "local-ipv4": {unique: true, path: "network/interface/0/ipv4/ipAddress/0/privateIpAddress"}, + "public-ipv4": {unique: true, path: "network/interface/0/ipv4/ipAddress/0/publicIpAddress"}, + "local-ipv6": {unique: true, path: "network/interface/0/ipv6/ipAddress/0/privateIpAddress"}, + "public-ipv6": {unique: true, path: "network/interface/0/ipv6/ipAddress/0/publicIpAddress"}, + "mac": {unique: true, path: "network/interface/0/macAddress"}, + } + + for k, attr := range keys { + resp, err := f.Get(attr.path, "text") + v := strings.TrimSpace(resp) + if err != nil { + return checkAzureError(err, f.logger, k) + } else if v == "" { + f.logger.Debug("read an empty value", "attribute", k) + continue + } + + // assume we want blank entries + key := "platform.azure." + strings.Replace(k, "/", ".", -1) + if attr.unique { + key = structs.UniqueNamespace(key) + } + response.AddAttribute(key, v) + } + + // copy over network specific information + if val, ok := response.Attributes["unique.platform.azure.local-ipv4"]; ok && val != "" { + response.AddAttribute("unique.network.ip-address", val) + } + + var tagList []AzureMetadataTag + value, err := f.Get("compute/tagsList", "json") + if err != nil { + return checkAzureError(err, f.logger, "tags") + } + if err := json.Unmarshal([]byte(value), &tagList); err != nil { + f.logger.Warn("error decoding instance tags", "error", err) + } + for _, tag := range tagList { + attr := "platform.azure.tag." + var key string + + // If the tag is namespaced as unique, we strip it from the tag and + // prepend to the whole attribute. + if structs.IsUniqueNamespace(tag.Name) { + tag.Name = strings.TrimPrefix(tag.Name, structs.NodeUniqueNamespace) + key = fmt.Sprintf("%s%s%s", structs.NodeUniqueNamespace, attr, tag.Name) + } else { + key = fmt.Sprintf("%s%s", attr, tag.Name) + } + + response.AddAttribute(key, tag.Value) + } + + // populate Links + if id, ok := response.Attributes["unique.platform.azure.id"]; ok { + response.AddLink("azure", id) + } + + response.Detected = true + return nil +} + +func (f *EnvAzureFingerprint) isAzure() bool { + v, err := f.Get("compute/azEnvironment", "text") + v = strings.TrimSpace(v) + return err == nil && v != "" +} diff --git a/client/fingerprint/env_azure_test.go b/client/fingerprint/env_azure_test.go new file mode 100644 index 000000000000..07b09dcbe693 --- /dev/null +++ b/client/fingerprint/env_azure_test.go @@ -0,0 +1,219 @@ +package fingerprint + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestAzureFingerprint_nonAzure(t *testing.T) { + os.Setenv("AZURE_ENV_URL", "http://127.0.0.1/metadata/instance/") + f := NewEnvAzureFingerprint(testlog.HCLogger(t)) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + request := &FingerprintRequest{Config: &config.Config{}, Node: node} + var response FingerprintResponse + err := f.Fingerprint(request, &response) + if err != nil { + t.Fatalf("err: %v", err) + } + + if response.Detected { + t.Fatalf("expected response to not be applicable") + } + + if len(response.Attributes) > 0 { + t.Fatalf("Should have zero attributes without test server") + } +} + +func testFingerprint_Azure(t *testing.T, withExternalIp bool) { + node := &structs.Node{ + Attributes: make(map[string]string), + } + + // configure mock server with fixture routes, data + routes := routes{} + if err := json.Unmarshal([]byte(AZURE_routes), &routes); err != nil { + t.Fatalf("Failed to unmarshal JSON in GCE ENV test: %s", err) + } + if withExternalIp { + networkEndpoint := &endpoint{ + Uri: "/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress", + ContentType: "text/plain", + Body: "104.44.55.66", + } + routes.Endpoints = append(routes.Endpoints, networkEndpoint) + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + value, ok := r.Header["Metadata"] + if !ok { + t.Fatal("Metadata not present in HTTP request header") + } + if value[0] != "true" { + t.Fatalf("Expected Metadata true, saw %s", value[0]) + } + + uavalue, ok := r.Header["User-Agent"] + if !ok { + t.Fatal("User-Agent not present in HTTP request header") + } + if !strings.Contains(uavalue[0], "Nomad/") { + t.Fatalf("Expected User-Agent to contain Nomad/, got %s", uavalue[0]) + } + + uri := r.RequestURI + if r.URL.RawQuery != "" { + uri = strings.Replace(uri, "?"+r.URL.RawQuery, "", 1) + } + + found := false + for _, e := range routes.Endpoints { + if uri == e.Uri { + w.Header().Set("Content-Type", e.ContentType) + fmt.Fprintln(w, e.Body) + found = true + } + } + + if !found { + w.WriteHeader(404) + } + })) + defer ts.Close() + os.Setenv("AZURE_ENV_URL", ts.URL+"/metadata/instance/") + f := NewEnvAzureFingerprint(testlog.HCLogger(t)) + + request := &FingerprintRequest{Config: &config.Config{}, Node: node} + var response FingerprintResponse + err := f.Fingerprint(request, &response) + if err != nil { + t.Fatalf("err: %v", err) + } + + if !response.Detected { + t.Fatalf("expected response to be applicable") + } + + keys := []string{ + "unique.platform.azure.id", + "unique.platform.azure.hostname", + "platform.azure.location", + "platform.azure.resource-group", + "platform.azure.scale-set", + "platform.azure.vm-size", + "unique.platform.azure.local-ipv4", + "unique.platform.azure.mac", + "platform.azure.tag.Environment", + "platform.azure.tag.abc", + "unique.platform.azure.tag.foo", + } + + for _, k := range keys { + assertNodeAttributeContains(t, response.Attributes, k) + } + + if len(response.Links) == 0 { + t.Fatalf("Empty links for Node in GCE Fingerprint test") + } + + // Make sure Links contains the GCE ID. + for _, k := range []string{"azure"} { + assertNodeLinksContains(t, response.Links, k) + } + + assertNodeAttributeEquals(t, response.Attributes, "unique.platform.azure.id", "13f56399-bd52-4150-9748-7190aae1ff21") + assertNodeAttributeEquals(t, response.Attributes, "unique.platform.azure.hostname", "demo01.internal") + assertNodeAttributeEquals(t, response.Attributes, "platform.azure.location", "eastus") + assertNodeAttributeEquals(t, response.Attributes, "platform.azure.resource-group", "myrg") + assertNodeAttributeEquals(t, response.Attributes, "platform.azure.scale-set", "nomad-clients") + assertNodeAttributeEquals(t, response.Attributes, "unique.platform.azure.local-ipv4", "10.1.0.4") + assertNodeAttributeEquals(t, response.Attributes, "unique.platform.azure.mac", "000D3AF806EC") + assertNodeAttributeEquals(t, response.Attributes, "platform.azure.tag.Environment", "Test") + assertNodeAttributeEquals(t, response.Attributes, "platform.azure.tag.abc", "def") + assertNodeAttributeEquals(t, response.Attributes, "unique.platform.azure.tag.foo", "true") + + if withExternalIp { + assertNodeAttributeEquals(t, response.Attributes, "unique.platform.azure.public-ipv4", "104.44.55.66") + } else if _, ok := response.Attributes["unique.platform.azure.public-ipv4"]; ok { + t.Fatal("unique.platform.azure.public-ipv4 is set without an external IP") + } + +} + +const AZURE_routes = ` +{ + "endpoints": [ + { + "uri": "/metadata/instance/compute/azEnvironment", + "content-type": "text/plain", + "body": "AzurePublicCloud" + }, + + { + "uri": "/metadata/instance/compute/location", + "content-type": "text/plain", + "body": "eastus" + }, + { + "uri": "/metadata/instance/compute/name", + "content-type": "text/plain", + "body": "demo01.internal" + }, + { + "uri": "/metadata/instance/compute/resourceGroupName", + "content-type": "text/plain", + "body": "myrg" + }, + { + "uri": "/metadata/instance/compute/vmId", + "content-type": "text/plain", + "body": "13f56399-bd52-4150-9748-7190aae1ff21" + }, + { + "uri": "/metadata/instance/compute/vmScaleSetName", + "content-type": "text/plain", + "body": "nomad-clients" + }, + { + "uri": "/metadata/instance/compute/vmSize", + "content-type": "text/plain", + "body": "Standard_A1_v2" + }, + { + "uri": "/metadata/instance/compute/tagsList", + "content-type": "application/json", + "body": "[{ \"name\":\"Environment\", \"value\":\"Test\"}, { \"name\":\"abc\", \"value\":\"def\"}, { \"name\":\"unique.foo\", \"value\":\"true\"}]" + }, + { + "uri": "/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress", + "content-type": "text/plain", + "body": "10.1.0.4" + }, + { + "uri": "/metadata/instance/network/interface/0/macAddress", + "content-type": "text/plain", + "body": "000D3AF806EC" + } + ] +} +` + +func TestFingerprint_AzureWithExternalIp(t *testing.T) { + testFingerprint_Azure(t, true) +} + +func TestFingerprint_AzureWithoutExternalIp(t *testing.T) { + testFingerprint_Azure(t, false) +} diff --git a/client/fingerprint/fingerprint.go b/client/fingerprint/fingerprint.go index d896b48e2240..8953de880e18 100644 --- a/client/fingerprint/fingerprint.go +++ b/client/fingerprint/fingerprint.go @@ -46,8 +46,9 @@ var ( // This should run after the host fingerprinters as they may override specific // node resources with more detailed information. envFingerprinters = map[string]Factory{ - "env_aws": NewEnvAWSFingerprint, - "env_gce": NewEnvGCEFingerprint, + "env_aws": NewEnvAWSFingerprint, + "env_gce": NewEnvGCEFingerprint, + "env_azure": NewEnvAzureFingerprint, } )