-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Refactor client fingerprinters to return a diff of node attributes #3781
Changes from 2 commits
5e8151d
a76a404
c21ac46
f5fc20a
ae889b4
e8aaa93
a9447ad
ba2ebbc
e5ccc55
15cb768
3202200
8049aa0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,6 +23,7 @@ import ( | |
"github.com/hashicorp/nomad/client/driver" | ||
"github.com/hashicorp/nomad/client/fingerprint" | ||
"github.com/hashicorp/nomad/client/stats" | ||
cstructs "github.com/hashicorp/nomad/client/structs" | ||
"github.com/hashicorp/nomad/client/vaultclient" | ||
"github.com/hashicorp/nomad/command/agent/consul" | ||
"github.com/hashicorp/nomad/helper" | ||
|
@@ -949,15 +950,31 @@ func (c *Client) fingerprint() error { | |
return err | ||
} | ||
|
||
c.configLock.Lock() | ||
applies, err := f.Fingerprint(c.config, c.config.Node) | ||
c.configLock.Unlock() | ||
request := &cstructs.FingerprintRequest{Config: c.config, Node: c.config.Node} | ||
response := &cstructs.FingerprintResponse{ | ||
Attributes: make(map[string]string, 0), | ||
Links: make(map[string]string, 0), | ||
Resources: &structs.Resources{}, | ||
} | ||
|
||
err = f.Fingerprint(request, response) | ||
if err != nil { | ||
return err | ||
} | ||
if applies { | ||
|
||
// if an attribute should be skipped, remove it from the list which we will | ||
// later apply to the node | ||
for _, e := range skipped { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These aren't attributes that are being skipped. These are whole fingerprinters so they won't have created any attribute that needs to be deleted. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah, only visible after expanding to get context that skipped is constructed from fingerprinter names. The diff made it look like it was mutating the output based on the skipped list. |
||
delete(response.Attributes, e) | ||
} | ||
|
||
for name := range response.Attributes { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The applied list is applied fingerprinters not the attributes. |
||
applied = append(applied, name) | ||
} | ||
|
||
// add the diff found from each fingerprinter | ||
c.updateNodeFromFingerprint(response) | ||
|
||
p, period := f.Periodic() | ||
if p { | ||
// TODO: If more periodic fingerprinters are added, then | ||
|
@@ -966,6 +983,7 @@ func (c *Client) fingerprint() error { | |
go c.fingerprintPeriodic(name, f, period) | ||
} | ||
} | ||
|
||
c.logger.Printf("[DEBUG] client: applied fingerprints %v", applied) | ||
if len(skipped) != 0 { | ||
c.logger.Printf("[DEBUG] client: fingerprint modules skipped due to white/blacklist: %v", skipped) | ||
|
@@ -979,11 +997,21 @@ func (c *Client) fingerprintPeriodic(name string, f fingerprint.Fingerprint, d t | |
for { | ||
select { | ||
case <-time.After(d): | ||
c.configLock.Lock() | ||
if _, err := f.Fingerprint(c.config, c.config.Node); err != nil { | ||
request := &cstructs.FingerprintRequest{Config: c.config, Node: c.config.Node} | ||
response := &cstructs.FingerprintResponse{ | ||
Attributes: make(map[string]string, 0), | ||
Links: make(map[string]string, 0), | ||
Resources: &structs.Resources{}, | ||
} | ||
|
||
err := f.Fingerprint(request, response) | ||
|
||
if err != nil { | ||
c.logger.Printf("[DEBUG] client: periodic fingerprinting for %v failed: %v", name, err) | ||
} else { | ||
c.updateNodeFromFingerprint(response) | ||
} | ||
c.configLock.Unlock() | ||
|
||
case <-c.shutdownCh: | ||
return | ||
} | ||
|
@@ -1017,16 +1045,30 @@ func (c *Client) setupDrivers() error { | |
if err != nil { | ||
return err | ||
} | ||
c.configLock.Lock() | ||
applies, err := d.Fingerprint(c.config, c.config.Node) | ||
c.configLock.Unlock() | ||
|
||
request := &cstructs.FingerprintRequest{Config: c.config, Node: c.config.Node} | ||
response := &cstructs.FingerprintResponse{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would prefer if it looked like:
You can then have methods on the response to be like:
Where it initializes the value. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good idea There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Alternatively, we could have a constructor which would initialize all of these values. I will add getter/setter methods that does initialization for each field, let me know if you think it is cleaner to just do initialization all at once in a constructor. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah the constructor could work but it is just idiomatic go to call an RPC like I showed, which is the direction we will be moving. |
||
Attributes: make(map[string]string, 0), | ||
Links: make(map[string]string, 0), | ||
Resources: &structs.Resources{}, | ||
} | ||
|
||
err = d.Fingerprint(request, response) | ||
if err != nil { | ||
return err | ||
} | ||
if applies { | ||
|
||
// remove attributes we are supposed to skip | ||
for _, attr := range skipped { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we don't care about thiese skipped attributes and are going to delete them from the response as soon as the fingerprint call is done, I would suggest pushing that logic one level down. You could have the FingerprintRequest also take a list of skipped(better name would be ignore) and have the implementors of The current approach works, but it is not well encapsulated - its doing something, then mutating the response before using it. |
||
delete(response.Attributes, attr) | ||
} | ||
|
||
for name := range response.Attributes { | ||
avail = append(avail, name) | ||
} | ||
|
||
c.updateNodeFromFingerprint(response) | ||
|
||
p, period := d.Periodic() | ||
if p { | ||
go c.fingerprintPeriodic(name, d, period) | ||
|
@@ -1035,6 +1077,7 @@ func (c *Client) setupDrivers() error { | |
} | ||
|
||
c.logger.Printf("[DEBUG] client: available drivers %v", avail) | ||
c.logger.Printf("[DEBUG] client: skipped attributes %v", skipped) | ||
|
||
if len(skipped) != 0 { | ||
c.logger.Printf("[DEBUG] client: drivers skipped due to white/blacklist: %v", skipped) | ||
|
@@ -1043,6 +1086,24 @@ func (c *Client) setupDrivers() error { | |
return nil | ||
} | ||
|
||
// updateNodeFromFingerprint updates the node with the result of | ||
// fingerprinting the node from the diff that was created | ||
func (c *Client) updateNodeFromFingerprint(response *cstructs.FingerprintResponse) { | ||
c.configLock.Lock() | ||
defer c.configLock.Unlock() | ||
for name, val := range response.Attributes { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How are you handling removing Attributes/Links? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As in fingerprinter_foo adds attribute = {"foo": "bar", "bam": "baz"} and then the next time it is called only returns {"foo": "bar"} because "bam" should be removed. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good point. One way we can handle this is if the attribute is equal to empty in the fingerprinter's response, we can delete this attribute from the node's attributes. That way we don't need to rely on the fingerprinter modifying the node directly. |
||
c.config.Node.Attributes[name] = val | ||
} | ||
|
||
// update node links and resources from the diff created from | ||
// fingerprinting | ||
for name, val := range response.Links { | ||
c.config.Node.Links[name] = val | ||
} | ||
|
||
c.config.Node.Resources.Merge(response.Resources) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I am fine with this strategy for now but this doesn't let a fingerprinter report a non-zero value and then set it to zero. The long term solution is that we pass in a different struct type that has all the fields as a pointer and as long as the fingerprinter sets the pointer we merge the value There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'll add that as a tech task to complete after this refactor. |
||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You should change the node watcher from checking periodically to check via a channel trigger and send it when this function is called: https://github.com/hashicorp/nomad/blob/master/client/client.go#L1588 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This will be in a follow up PR. |
||
|
||
// retryIntv calculates a retry interval value given the base | ||
func (c *Client) retryIntv(base time.Duration) time.Duration { | ||
if c.config.DevMode { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,8 @@ | ||
package driver | ||
|
||
import ( | ||
"github.com/hashicorp/nomad/client/config" | ||
cstructs "github.com/hashicorp/nomad/client/structs" | ||
"github.com/hashicorp/nomad/helper" | ||
"github.com/hashicorp/nomad/nomad/structs" | ||
"golang.org/x/sys/unix" | ||
) | ||
|
||
|
@@ -13,28 +12,26 @@ const ( | |
execDriverAttr = "driver.exec" | ||
) | ||
|
||
func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { | ||
func (d *ExecDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error { | ||
// Only enable if cgroups are available and we are root | ||
if !cgroupsMounted(node) { | ||
if !cgroupsMounted(req.Node) { | ||
if d.fingerprintSuccess == nil || *d.fingerprintSuccess { | ||
d.logger.Printf("[DEBUG] driver.exec: cgroups unavailable, disabling") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you make this an INFO level log |
||
} | ||
d.fingerprintSuccess = helper.BoolToPtr(false) | ||
delete(node.Attributes, execDriverAttr) | ||
return false, nil | ||
return nil | ||
} else if unix.Geteuid() != 0 { | ||
if d.fingerprintSuccess == nil || *d.fingerprintSuccess { | ||
d.logger.Printf("[DEBUG] driver.exec: must run as root user, disabling") | ||
} | ||
delete(node.Attributes, execDriverAttr) | ||
d.fingerprintSuccess = helper.BoolToPtr(false) | ||
return false, nil | ||
return nil | ||
} | ||
|
||
if d.fingerprintSuccess == nil || !*d.fingerprintSuccess { | ||
d.logger.Printf("[DEBUG] driver.exec: exec driver is enabled") | ||
} | ||
node.Attributes[execDriverAttr] = "1" | ||
resp.Attributes[execDriverAttr] = "1" | ||
d.fingerprintSuccess = helper.BoolToPtr(true) | ||
return true, nil | ||
return nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't believe you can get rid of the lock yet. This is a shared data structure that is being mutated/read concurrently. Apples to all places you are doing a fingerprint.