Merge pull request #12015 from kevinschoonover/main

client/fingerprint: add digitalocean fingerprinter

client/fingerprint/env_digitalocean.go

@@ -0,0 +1,163 @@
+package fingerprint
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	cleanhttp "github.com/hashicorp/go-cleanhttp"
+	log "github.com/hashicorp/go-hclog"
+
+	"github.com/hashicorp/nomad/helper/useragent"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+const (
+	// DigitalOceanMetadataURL is where the DigitalOcean metadata api normally resides.
+	// https://docs.digitalocean.com/products/droplets/how-to/retrieve-droplet-metadata/#how-to-retrieve-droplet-metadata
+	DigitalOceanMetadataURL = "http://169.254.169.254/metadata/v1/"
+
+	// DigitalOceanMetadataTimeout is the timeout used when contacting the DigitalOcean metadata
+	// services.
+	DigitalOceanMetadataTimeout = 2 * time.Second
+)
+
+type DigitalOceanMetadataPair struct {
+	path   string
+	unique bool
+}
+
+// EnvDigitalOceanFingerprint is used to fingerprint DigitalOcean metadata
+type EnvDigitalOceanFingerprint struct {
+	StaticFingerprinter
+	client      *http.Client
+	logger      log.Logger
+	metadataURL string
+}
+
+// NewEnvDigitalOceanFingerprint is used to create a fingerprint from DigitalOcean metadata
+func NewEnvDigitalOceanFingerprint(logger log.Logger) Fingerprint {
+	// Read the internal metadata URL from the environment, allowing test files to
+	// provide their own
+	metadataURL := os.Getenv("DO_ENV_URL")
+	if metadataURL == "" {
+		metadataURL = DigitalOceanMetadataURL
+	}
+
+	// assume 2 seconds is enough time for inside DigitalOcean network
+	client := &http.Client{
+		Timeout:   DigitalOceanMetadataTimeout,
+		Transport: cleanhttp.DefaultTransport(),
+	}
+
+	return &EnvDigitalOceanFingerprint{
+		client:      client,
+		logger:      logger.Named("env_digitalocean"),
+		metadataURL: metadataURL,
+	}
+}
+
+func (f *EnvDigitalOceanFingerprint) Get(attribute string, format string) (string, error) {
+	reqURL := f.metadataURL + attribute
+	parsedURL, err := url.Parse(reqURL)
+	if err != nil {
+		return "", err
+	}
+
+	req := &http.Request{
+		Method: http.MethodGet,
+		URL:    parsedURL,
+		Header: http.Header{
+			"User-Agent": []string{useragent.String()},
+		},
+	}
+
+	res, err := f.client.Do(req)
+	if err != nil {
+		f.logger.Debug("failed to request metadata", "attribute", attribute, "error", err)
+		return "", err
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		f.logger.Error("failed to read metadata", "attribute", attribute, "error", err, "resp_code", res.StatusCode)
+		return "", err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		f.logger.Debug("could not read value for attribute", "attribute", attribute, "resp_code", res.StatusCode)
+		return "", fmt.Errorf("error reading attribute %s. digitalocean metadata api returned an error: resp_code: %d, resp_body: %s", attribute, res.StatusCode, body)
+	}
+
+	return string(body), nil
+}
+
+func (f *EnvDigitalOceanFingerprint) Fingerprint(request *FingerprintRequest, response *FingerprintResponse) error {
+	cfg := request.Config
+
+	// Check if we should tighten the timeout
+	if cfg.ReadBoolDefault(TightenNetworkTimeoutsConfig, false) {
+		f.client.Timeout = 1 * time.Millisecond
+	}
+
+	if !f.isDigitalOcean() {
+		return nil
+	}
+
+	// Keys and whether they should be namespaced as unique. Any key whose value
+	// uniquely identifies a node, such as ip, should be marked as unique. When
+	// marked as unique, the key isn't included in the computed node class.
+	keys := map[string]DigitalOceanMetadataPair{
+		"id":           {unique: true, path: "id"},
+		"hostname":     {unique: true, path: "hostname"},
+		"region":       {unique: false, path: "region"},
+		"private-ipv4": {unique: true, path: "interfaces/private/0/ipv4/address"},
+		"public-ipv4":  {unique: true, path: "interfaces/public/0/ipv4/address"},
+		"private-ipv6": {unique: true, path: "interfaces/private/0/ipv6/address"},
+		"public-ipv6":  {unique: true, path: "interfaces/public/0/ipv6/address"},
+		"mac":          {unique: true, path: "interfaces/public/0/mac"},
+	}
+
+	for k, attr := range keys {
+		resp, err := f.Get(attr.path, "text")
+		v := strings.TrimSpace(resp)
+		if err != nil {
+			f.logger.Warn("failed to read attribute", "attribute", k, "err", err)
+			continue
+		} else if v == "" {
+			f.logger.Debug("read an empty value", "attribute", k)
+			continue
+		}
+
+		// assume we want blank entries
+		key := "platform.digitalocean." + strings.ReplaceAll(k, "/", ".")
+		if attr.unique {
+			key = structs.UniqueNamespace(key)
+		}
+		response.AddAttribute(key, v)
+	}
+
+	// copy over network specific information
+	if val, ok := response.Attributes["unique.platform.digitalocean.local-ipv4"]; ok && val != "" {
+		response.AddAttribute("unique.network.ip-address", val)
+	}
+
+	// populate Links
+	if id, ok := response.Attributes["unique.platform.digitalocean.id"]; ok {
+		response.AddLink("digitalocean", id)
+	}
+
+	response.Detected = true
+	return nil
+}
+
+func (f *EnvDigitalOceanFingerprint) isDigitalOcean() bool {
+	v, err := f.Get("region", "text")
+	v = strings.TrimSpace(v)
+	return err == nil && v != ""
+}

client/fingerprint/env_digitalocean_test.go

@@ -0,0 +1,167 @@
+package fingerprint
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDigitalOceanFingerprint_nonDigitalOcean(t *testing.T) {
+	os.Setenv("DO_ENV_URL", "http://127.0.0.1/metadata/v1/")
+	f := NewEnvDigitalOceanFingerprint(testlog.HCLogger(t))
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+
+	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
+	var response FingerprintResponse
+	err := f.Fingerprint(request, &response)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if response.Detected {
+		t.Fatalf("expected response to not be applicable")
+	}
+
+	if len(response.Attributes) > 0 {
+		t.Fatalf("Should have zero attributes without test server")
+	}
+}
+
+func TestFingerprint_DigitalOcean(t *testing.T) {
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+
+	// configure mock server with fixture routes, data
+	routes := routes{}
+	if err := json.Unmarshal([]byte(DO_routes), &routes); err != nil {
+		t.Fatalf("Failed to unmarshal JSON in DO ENV test: %s", err)
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		value, ok := r.Header["Metadata"]
+		if !ok {
+			t.Fatal("Metadata not present in HTTP request header")
+		}
+		if value[0] != "true" {
+			t.Fatalf("Expected Metadata true, saw %s", value[0])
+		}
+
+		uavalue, ok := r.Header["User-Agent"]
+		if !ok {
+			t.Fatal("User-Agent not present in HTTP request header")
+		}
+		if !strings.Contains(uavalue[0], "Nomad/") {
+			t.Fatalf("Expected User-Agent to contain Nomad/, got %s", uavalue[0])
+		}
+
+		uri := r.RequestURI
+		if r.URL.RawQuery != "" {
+			uri = strings.Replace(uri, "?"+r.URL.RawQuery, "", 1)
+		}
+
+		found := false
+		for _, e := range routes.Endpoints {
+			if uri == e.Uri {
+				w.Header().Set("Content-Type", e.ContentType)
+				fmt.Fprintln(w, e.Body)
+				found = true
+			}
+		}
+
+		if !found {
+			w.WriteHeader(404)
+		}
+	}))
+	defer ts.Close()
+	os.Setenv("DO_ENV_URL", ts.URL+"/metadata/v1/")
+	f := NewEnvDigitalOceanFingerprint(testlog.HCLogger(t))
+
+	request := &FingerprintRequest{Config: &config.Config{}, Node: node}
+	var response FingerprintResponse
+	err := f.Fingerprint(request, &response)
+	assert.NoError(t, err)
+	assert.True(t, response.Detected, "expected response to be applicable")
+
+	keys := []string{
+		"unique.platform.digitalocean.id",
+		"unique.platform.digitalocean.hostname",
+		"platform.digitalocean.region",
+		"unique.platform.digitalocean.private-ipv4",
+		"unique.platform.digitalocean.public-ipv4",
+		"unique.platform.digitalocean.public-ipv6",
+		"unique.platform.digitalocean.mac",
+	}
+
+	for _, k := range keys {
+		assertNodeAttributeContains(t, response.Attributes, k)
+	}
+
+	assert.NotEmpty(t, response.Links, "Empty links for Node in DO Fingerprint test")
+
+	// Make sure Links contains the DO ID.
+	for _, k := range []string{"digitalocean"} {
+		assertNodeLinksContains(t, response.Links, k)
+	}
+
+	assertNodeAttributeEquals(t, response.Attributes, "unique.platform.digitalocean.id", "13f56399-bd52-4150-9748-7190aae1ff21")
+	assertNodeAttributeEquals(t, response.Attributes, "unique.platform.digitalocean.hostname", "demo01.internal")
+	assertNodeAttributeEquals(t, response.Attributes, "platform.digitalocean.region", "sfo3")
+	assertNodeAttributeEquals(t, response.Attributes, "unique.platform.digitalocean.private-ipv4", "10.1.0.4")
+	assertNodeAttributeEquals(t, response.Attributes, "unique.platform.digitalocean.mac", "000D3AF806EC")
+	assertNodeAttributeEquals(t, response.Attributes, "unique.platform.digitalocean.public-ipv4", "100.100.100.100")
+	assertNodeAttributeEquals(t, response.Attributes, "unique.platform.digitalocean.public-ipv6", "c99c:8ac5:3112:204b:48b0:41aa:e085:d11a")
+}
+
+const DO_routes = `
+{
+  "endpoints": [
+	{
+	  "uri": "/metadata/v1/region",
+	  "content-type": "text/plain",
+      "body": "sfo3"
+	},
+	{
+		"uri": "/metadata/v1/hostname",
+		"content-type": "text/plain",
+		"body": "demo01.internal"
+	},
+	{
+		"uri": "/metadata/v1/id",
+		"content-type": "text/plain",
+		"body": "13f56399-bd52-4150-9748-7190aae1ff21"
+	},
+	{
+		"uri": "/metadata/v1/interfaces/private/0/ipv4/address",
+		"content-type": "text/plain",
+		"body": "10.1.0.4"
+	},
+	{
+		"uri": "/metadata/v1/interfaces/public/0/mac",
+		"content-type": "text/plain",
+		"body": "000D3AF806EC"
+	},
+  {
+		"uri": "/metadata/v1/interfaces/public/0/ipv4/address",
+		"content-type": "text/plain",
+		"body": "100.100.100.100"
+	},
+  {
+		"uri": "/metadata/v1/interfaces/public/0/ipv6/address",
+		"content-type": "text/plain",
+		"body": "c99c:8ac5:3112:204b:48b0:41aa:e085:d11a"
+	}
+  ]
+}
+`

client/fingerprint/fingerprint.go

@@ -46,9 +46,10 @@ var (
 	// This should run after the host fingerprinters as they may override specific
 	// node resources with more detailed information.
 	envFingerprinters = map[string]Factory{
-		"env_aws":   NewEnvAWSFingerprint,
-		"env_gce":   NewEnvGCEFingerprint,
-		"env_azure": NewEnvAzureFingerprint,
+		"env_aws":          NewEnvAWSFingerprint,
+		"env_gce":          NewEnvGCEFingerprint,
+		"env_azure":        NewEnvAzureFingerprint,
+		"env_digitalocean": NewEnvDigitalOceanFingerprint,
 	}
 )