From dc3c0b134ac097aa8c5b8ddf7c472f5e59967863 Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Sat, 20 Nov 2021 10:04:15 -0500 Subject: [PATCH 01/40] feat: add status filter to allocations --- ui/app/controllers/jobs/job/allocations.js | 45 +++++++-- ui/app/templates/jobs/job/allocations.hbs | 102 ++++++++++++++++----- 2 files changed, 116 insertions(+), 31 deletions(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index a01c199510d2..a8fc3dad6f8f 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -4,14 +4,15 @@ import { action, computed } from '@ember/object'; import Sortable from 'nomad-ui/mixins/sortable'; import Searchable from 'nomad-ui/mixins/searchable'; import WithNamespaceResetting from 'nomad-ui/mixins/with-namespace-resetting'; +import { serialize, deserializedQueryParam as selection } from 'nomad-ui/utils/qp-serialize'; import classic from 'ember-classic-decorator'; @classic export default class AllocationsController extends Controller.extend( - Sortable, - Searchable, - WithNamespaceResetting - ) { + Sortable, + Searchable, + WithNamespaceResetting +) { queryParams = [ { currentPage: 'page', @@ -25,8 +26,12 @@ export default class AllocationsController extends Controller.extend( { sortDescending: 'desc', }, + { + qpStatus: 'status', + }, ]; + qpStatus = ''; currentPage = 1; pageSize = 25; @@ -40,11 +45,24 @@ export default class AllocationsController extends Controller.extend( return ['shortId', 'name', 'taskGroupName']; } - @computed('model.allocations.[]') + @computed('model.allocations.[]', 'selectionStatus') get allocations() { - return this.get('model.allocations') || []; + const allocations = this.get('model.allocations') || []; + const { selectionStatus } = this; + + if (!allocations.length) return allocations; + + return allocations.filter(alloc => { + if (selectionStatus.length && !selectionStatus.includes(alloc.status)) { + return false; + } + + return true; + }); } + @selection('qpStatus') selectionStatus; + @alias('allocations') listToSort; @alias('listSorted') listToSearch; @alias('listSearched') sortedAllocations; @@ -53,4 +71,19 @@ export default class AllocationsController extends Controller.extend( gotoAllocation(allocation) { this.transitionToRoute('allocations.allocation', allocation); } + + get optionsAllocationStatus() { + return [ + { key: 'queued', label: 'Queued' }, + { key: 'starting', label: 'Starting' }, + { key: 'running', label: 'Running' }, + { key: 'complete', label: 'Complete' }, + { key: 'failed', label: 'Failed' }, + { key: 'lost', label: 'Lost' }, + ]; + } + + setFacetQueryParam(queryParam, selection) { + this.set(queryParam, serialize(selection)); + } } diff --git a/ui/app/templates/jobs/job/allocations.hbs b/ui/app/templates/jobs/job/allocations.hbs index e14452e483d3..7d62da118577 100644 --- a/ui/app/templates/jobs/job/allocations.hbs +++ b/ui/app/templates/jobs/job/allocations.hbs @@ -1,14 +1,26 @@ {{page-title "Job " this.job.name " allocations"}}
- {{#if this.allocations.length}} -
-
+ {{#if this.model.allocations.length}} +
+
+ @placeholder="Search allocations..." + /> +
+
+
+ +
{{#if this.sortedAllocations}} @@ -16,40 +28,69 @@ @source={{this.sortedAllocations}} @size={{this.pageSize}} @page={{this.currentPage}} - @class="allocations" as |p|> + @class="allocations" as |p| + > + @class="with-foot" as |t| + > - ID - Task Group - Created - Modified - Status - Version - Client - Volume - CPU - Memory + + ID + + + Task Group + + + Created + + + Modified + + + Status + + + Version + + + Client + + + Volume + + + CPU + + + Memory + + @onClick={{action "gotoAllocation" row.model}} + />
@@ -57,17 +98,28 @@ {{else}}
-

No Matches

-

No allocations match the term {{this.searchTerm}}

+

+ No Matches +

+

+ No allocations match the term + + {{this.searchTerm}} + +

{{/if}} {{else}}
-

No Allocations

-

No allocations have been placed.

+

+ No Allocations +

+

+ No allocations have been placed. +

{{/if}} -
+ \ No newline at end of file From 14206f4c4f1f2db2278fcdb56be2e5c4070d5b25 Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Sat, 20 Nov 2021 10:21:28 -0500 Subject: [PATCH 02/40] feat: add client status filter --- ui/app/controllers/jobs/job/allocations.js | 28 ++++++++++++++++++++-- ui/app/templates/jobs/job/allocations.hbs | 7 ++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index a8fc3dad6f8f..8aa14e1e7954 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -1,6 +1,9 @@ +/* eslint-disable ember/no-incorrect-calls-with-inline-anonymous-functions */ import { alias } from '@ember/object/computed'; import Controller from '@ember/controller'; import { action, computed } from '@ember/object'; +import { scheduleOnce } from '@ember/runloop'; +import intersection from 'lodash.intersection'; import Sortable from 'nomad-ui/mixins/sortable'; import Searchable from 'nomad-ui/mixins/searchable'; import WithNamespaceResetting from 'nomad-ui/mixins/with-namespace-resetting'; @@ -29,9 +32,13 @@ export default class AllocationsController extends Controller.extend( { qpStatus: 'status', }, + { + qpClient: 'client', + }, ]; qpStatus = ''; + qpClient = ''; currentPage = 1; pageSize = 25; @@ -45,10 +52,10 @@ export default class AllocationsController extends Controller.extend( return ['shortId', 'name', 'taskGroupName']; } - @computed('model.allocations.[]', 'selectionStatus') + @computed('model.allocations.[]', 'selectionStatus', 'selectionClient') get allocations() { const allocations = this.get('model.allocations') || []; - const { selectionStatus } = this; + const { selectionStatus, selectionClient } = this; if (!allocations.length) return allocations; @@ -56,12 +63,16 @@ export default class AllocationsController extends Controller.extend( if (selectionStatus.length && !selectionStatus.includes(alloc.status)) { return false; } + if (selectionClient.length && !selectionClient.includes(alloc.get('node.shortId'))) { + return false; + } return true; }); } @selection('qpStatus') selectionStatus; + @selection('qpClient') selectionClient; @alias('allocations') listToSort; @alias('listSorted') listToSearch; @@ -83,6 +94,19 @@ export default class AllocationsController extends Controller.extend( ]; } + @computed('model.allocations.[]', 'selectionClient') + get optionsClients() { + const clients = Array.from(new Set(this.model.allocations.mapBy('node.shortId'))).compact(); + + // Update query param when the list of clients changes. + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpClient', serialize(intersection(clients, this.selectionClient))); + }); + + return clients.sort().map(dc => ({ key: dc, label: dc })); + } + setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); } diff --git a/ui/app/templates/jobs/job/allocations.hbs b/ui/app/templates/jobs/job/allocations.hbs index 7d62da118577..320575a519f5 100644 --- a/ui/app/templates/jobs/job/allocations.hbs +++ b/ui/app/templates/jobs/job/allocations.hbs @@ -20,6 +20,13 @@ @selection={{this.selectionStatus}} @onSelect={{action this.setFacetQueryParam "qpStatus"}} /> + From c351e68bed679f3ca205b642ccc694fff8962c3c Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Sat, 20 Nov 2021 10:30:48 -0500 Subject: [PATCH 03/40] feat: add taskgroup filter to alloc --- ui/app/controllers/jobs/job/allocations.js | 25 ++++++++++++++++++++-- ui/app/templates/jobs/job/allocations.hbs | 7 ++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index 8aa14e1e7954..15ba86319316 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -35,10 +35,14 @@ export default class AllocationsController extends Controller.extend( { qpClient: 'client', }, + { + qpTaskGroup: 'taskGroup', + }, ]; qpStatus = ''; qpClient = ''; + qpTaskGroup = ''; currentPage = 1; pageSize = 25; @@ -52,10 +56,10 @@ export default class AllocationsController extends Controller.extend( return ['shortId', 'name', 'taskGroupName']; } - @computed('model.allocations.[]', 'selectionStatus', 'selectionClient') + @computed('model.allocations.[]', 'selectionStatus', 'selectionClient', 'selectionTaskGroup') get allocations() { const allocations = this.get('model.allocations') || []; - const { selectionStatus, selectionClient } = this; + const { selectionStatus, selectionClient, selectionTaskGroup } = this; if (!allocations.length) return allocations; @@ -66,6 +70,9 @@ export default class AllocationsController extends Controller.extend( if (selectionClient.length && !selectionClient.includes(alloc.get('node.shortId'))) { return false; } + if (selectionTaskGroup.length && !selectionTaskGroup.includes(alloc.taskGroupName)) { + return false; + } return true; }); @@ -73,6 +80,7 @@ export default class AllocationsController extends Controller.extend( @selection('qpStatus') selectionStatus; @selection('qpClient') selectionClient; + @selection('qpTaskGroup') selectionTaskGroup; @alias('allocations') listToSort; @alias('listSorted') listToSearch; @@ -107,6 +115,19 @@ export default class AllocationsController extends Controller.extend( return clients.sort().map(dc => ({ key: dc, label: dc })); } + @computed('model.allocations.[]', 'selectionTaskGroup') + get optionsTaskGroups() { + const taskGroups = Array.from(new Set(this.model.allocations.mapBy('taskGroupName'))).compact(); + + // Update query param when the list of clients changes. + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpTaskGroup', serialize(intersection(taskGroups, this.selectionTaskGroup))); + }); + + return taskGroups.sort().map(dc => ({ key: dc, label: dc })); + } + setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); } diff --git a/ui/app/templates/jobs/job/allocations.hbs b/ui/app/templates/jobs/job/allocations.hbs index 320575a519f5..2e164ad899f4 100644 --- a/ui/app/templates/jobs/job/allocations.hbs +++ b/ui/app/templates/jobs/job/allocations.hbs @@ -27,6 +27,13 @@ @selection={{this.selectionClient}} @onSelect={{action this.setFacetQueryParam "qpClient"}} /> + From c3f14395f8b6a8c0530d58b542e7d1ddac111b8e Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Sat, 20 Nov 2021 10:49:31 -0500 Subject: [PATCH 04/40] disable eslint for indentation --- ui/app/controllers/jobs/job/allocations.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index 15ba86319316..d8e63f201df9 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -12,6 +12,7 @@ import classic from 'ember-classic-decorator'; @classic export default class AllocationsController extends Controller.extend( + /*eslint-disable indent */ Sortable, Searchable, WithNamespaceResetting From 727ca28212da834113a59a80900456da1af38f28 Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Sat, 20 Nov 2021 11:22:48 -0500 Subject: [PATCH 05/40] feat: add filter client allocations table --- ui/app/controllers/clients/client/index.js | 35 ++++++++++++++++++++-- ui/app/templates/clients/client/index.hbs | 10 ++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/ui/app/controllers/clients/client/index.js b/ui/app/controllers/clients/client/index.js index 9b0121e088d9..5ebb7e03a78a 100644 --- a/ui/app/controllers/clients/client/index.js +++ b/ui/app/controllers/clients/client/index.js @@ -7,6 +7,7 @@ import { task } from 'ember-concurrency'; import Sortable from 'nomad-ui/mixins/sortable'; import Searchable from 'nomad-ui/mixins/searchable'; import messageFromAdapterError from 'nomad-ui/utils/message-from-adapter-error'; +import { serialize, deserializedQueryParam as selection } from 'nomad-ui/utils/qp-serialize'; import classic from 'ember-classic-decorator'; @classic @@ -27,11 +28,15 @@ export default class ClientController extends Controller.extend(Sortable, Search { onlyPreemptions: 'preemptions', }, + { + qpStatus: 'status', + }, ]; // Set in the route flagAsDraining = false; + qpStatus = ''; currentPage = 1; pageSize = 8; @@ -45,15 +50,25 @@ export default class ClientController extends Controller.extend(Sortable, Search onlyPreemptions = false; - @computed('model.allocations.[]', 'preemptions.[]', 'onlyPreemptions') + @computed('model.allocations.[]', 'preemptions.[]', 'onlyPreemptions', 'selectionStatus') get visibleAllocations() { - return this.onlyPreemptions ? this.preemptions : this.model.allocations; + const allocations = this.onlyPreemptions ? this.preemptions : this.model.allocations; + const { selectionStatus } = this; + + return allocations.filter(alloc => { + if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { + return false; + } + return true; + }); } @alias('visibleAllocations') listToSort; @alias('listSorted') listToSearch; @alias('listSearched') sortedAllocations; + @selection('qpStatus') selectionStatus; + eligibilityError = null; stopDrainError = null; drainError = null; @@ -147,4 +162,20 @@ export default class ClientController extends Controller.extend(Sortable, Search const error = messageFromAdapterError(err) || 'Could not run drain'; this.set('drainError', error); } + + get optionsAllocationStatus() { + return [ + { key: 'queued', label: 'Queued' }, + { key: 'starting', label: 'Starting' }, + { key: 'running', label: 'Running' }, + { key: 'complete', label: 'Complete' }, + { key: 'failed', label: 'Failed' }, + { key: 'lost', label: 'Lost' }, + ]; + } + + @action + setFacetQueryParam(queryParam, selection) { + this.set(queryParam, serialize(selection)); + } } diff --git a/ui/app/templates/clients/client/index.hbs b/ui/app/templates/clients/client/index.hbs index 320bf7a335a1..04907498d6e1 100644 --- a/ui/app/templates/clients/client/index.hbs +++ b/ui/app/templates/clients/client/index.hbs @@ -299,7 +299,15 @@ @onChange={{action this.resetPagination}} @placeholder="Search allocations..." @class="is-inline pull-right" - @inputClass="is-compact" /> + @inputClass="is-compact" + /> +
Date: Sat, 20 Nov 2021 12:18:02 -0500 Subject: [PATCH 06/40] feat: add taskgroup filter to alloc table --- ui/app/controllers/clients/client/index.js | 34 ++++++++++++++++++++-- ui/app/templates/clients/client/index.hbs | 7 +++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/ui/app/controllers/clients/client/index.js b/ui/app/controllers/clients/client/index.js index 5ebb7e03a78a..a1d6f37b0ebc 100644 --- a/ui/app/controllers/clients/client/index.js +++ b/ui/app/controllers/clients/client/index.js @@ -1,9 +1,12 @@ /* eslint-disable ember/no-observers */ +/* eslint-disable ember/no-incorrect-calls-with-inline-anonymous-functions */ import { alias } from '@ember/object/computed'; import Controller from '@ember/controller'; import { action, computed } from '@ember/object'; import { observes } from '@ember-decorators/object'; +import { scheduleOnce } from '@ember/runloop'; import { task } from 'ember-concurrency'; +import intersection from 'lodash.intersection'; import Sortable from 'nomad-ui/mixins/sortable'; import Searchable from 'nomad-ui/mixins/searchable'; import messageFromAdapterError from 'nomad-ui/utils/message-from-adapter-error'; @@ -31,12 +34,16 @@ export default class ClientController extends Controller.extend(Sortable, Search { qpStatus: 'status', }, + { + qpTaskGroup: 'taskGroup', + }, ]; // Set in the route flagAsDraining = false; qpStatus = ''; + qpTaskGroup = ''; currentPage = 1; pageSize = 8; @@ -50,15 +57,24 @@ export default class ClientController extends Controller.extend(Sortable, Search onlyPreemptions = false; - @computed('model.allocations.[]', 'preemptions.[]', 'onlyPreemptions', 'selectionStatus') + @computed( + 'model.allocations.[]', + 'preemptions.[]', + 'onlyPreemptions', + 'selectionStatus', + 'selectionTaskGroup' + ) get visibleAllocations() { const allocations = this.onlyPreemptions ? this.preemptions : this.model.allocations; - const { selectionStatus } = this; + const { selectionStatus, selectionTaskGroup } = this; return allocations.filter(alloc => { if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { return false; } + if (selectionTaskGroup.length && !selectionTaskGroup.includes(alloc.taskGroupName)) { + return false; + } return true; }); } @@ -68,6 +84,7 @@ export default class ClientController extends Controller.extend(Sortable, Search @alias('listSearched') sortedAllocations; @selection('qpStatus') selectionStatus; + @selection('qpTaskGroup') selectionTaskGroup; eligibilityError = null; stopDrainError = null; @@ -174,6 +191,19 @@ export default class ClientController extends Controller.extend(Sortable, Search ]; } + @computed('model.allocations.[]', 'selectionTaskGroup') + get optionsTaskGroups() { + const taskGroups = Array.from(new Set(this.model.allocations.mapBy('taskGroupName'))).compact(); + + // Update query param when the list of clients changes. + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpTaskGroup', serialize(intersection(taskGroups, this.selectionTaskGroup))); + }); + + return taskGroups.sort().map(dc => ({ key: dc, label: dc })); + } + @action setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); diff --git a/ui/app/templates/clients/client/index.hbs b/ui/app/templates/clients/client/index.hbs index 04907498d6e1..e9f64dee00f2 100644 --- a/ui/app/templates/clients/client/index.hbs +++ b/ui/app/templates/clients/client/index.hbs @@ -308,6 +308,13 @@ @selection={{this.selectionStatus}} @onSelect={{action "setFacetQueryParam" "qpStatus"}} /> +
Date: Sat, 20 Nov 2021 14:48:28 -0500 Subject: [PATCH 07/40] fix: re-order multiselect and search boxes --- ui/app/styles/components/boxed-section.scss | 9 +++++ ui/app/templates/clients/client/index.hbs | 44 +++++++++++---------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/ui/app/styles/components/boxed-section.scss b/ui/app/styles/components/boxed-section.scss index bcebd868f410..901dc75036cf 100644 --- a/ui/app/styles/components/boxed-section.scss +++ b/ui/app/styles/components/boxed-section.scss @@ -19,6 +19,15 @@ margin-left: auto; } + .is-subsection { + display: flex; + align-items: baseline; + + .is-padded { + padding: 0em 0em 0em 1em; + } + } + .is-fixed-width { display: inline-block; width: 8em; diff --git a/ui/app/templates/clients/client/index.hbs b/ui/app/templates/clients/client/index.hbs index e9f64dee00f2..3ccb8de5d6f2 100644 --- a/ui/app/templates/clients/client/index.hbs +++ b/ui/app/templates/clients/client/index.hbs @@ -294,27 +294,29 @@ {{/if}}
- - - +
+ + + +
Date: Sat, 20 Nov 2021 15:07:27 -0500 Subject: [PATCH 08/40] feat: add filters to alloc table in task group view --- ui/app/controllers/jobs/job/task-group.js | 62 ++++++++++++++++++++++- ui/app/templates/jobs/job/task-group.hbs | 30 ++++++++--- 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/ui/app/controllers/jobs/job/task-group.js b/ui/app/controllers/jobs/job/task-group.js index a85c97d30b2c..b0fdb6d58cc1 100644 --- a/ui/app/controllers/jobs/job/task-group.js +++ b/ui/app/controllers/jobs/job/task-group.js @@ -1,10 +1,14 @@ +/* eslint-disable ember/no-incorrect-calls-with-inline-anonymous-functions */ import { inject as service } from '@ember/service'; import { alias, readOnly } from '@ember/object/computed'; import Controller from '@ember/controller'; import { action, computed, get } from '@ember/object'; +import { scheduleOnce } from '@ember/runloop'; +import intersection from 'lodash.intersection'; import Sortable from 'nomad-ui/mixins/sortable'; import Searchable from 'nomad-ui/mixins/searchable'; import WithNamespaceResetting from 'nomad-ui/mixins/with-namespace-resetting'; +import { serialize, deserializedQueryParam as selection } from 'nomad-ui/utils/qp-serialize'; import classic from 'ember-classic-decorator'; @classic @@ -29,11 +33,19 @@ export default class TaskGroupController extends Controller.extend( { sortDescending: 'desc', }, + { + qpStatus: 'status', + }, + { + qpClient: 'client', + }, ]; currentPage = 1; @readOnly('userSettings.pageSize') pageSize; + qpStatus = ''; + qpClient = ''; sortProperty = 'modifyIndex'; sortDescending = true; @@ -42,15 +54,32 @@ export default class TaskGroupController extends Controller.extend( return ['shortId', 'name']; } - @computed('model.allocations.[]') + @computed('model.allocations.[]', 'selectionStatus', 'selectionClient') get allocations() { - return this.get('model.allocations') || []; + const allocations = this.get('model.allocations') || []; + const { selectionStatus, selectionClient } = this; + + if (!allocations.length) return allocations; + + return allocations.filter(alloc => { + if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { + return false; + } + if (selectionClient.length && !selectionClient.includes(alloc.get('node.shortId'))) { + return false; + } + + return true; + }); } @alias('allocations') listToSort; @alias('listSorted') listToSearch; @alias('listSearched') sortedAllocations; + @selection('qpStatus') selectionStatus; + @selection('qpClient') selectionClient; + @computed('model.scaleState.events.@each.time', function() { const events = get(this, 'model.scaleState.events'); if (events) { @@ -83,4 +112,33 @@ export default class TaskGroupController extends Controller.extend( scaleTaskGroup(count) { return this.model.scale(count); } + + get optionsAllocationStatus() { + return [ + { key: 'queued', label: 'Queued' }, + { key: 'starting', label: 'Starting' }, + { key: 'running', label: 'Running' }, + { key: 'complete', label: 'Complete' }, + { key: 'failed', label: 'Failed' }, + { key: 'lost', label: 'Lost' }, + ]; + } + + @computed('model.allocations.[]', 'selectionClient') + get optionsClients() { + const clients = Array.from(new Set(this.model.allocations.mapBy('node.shortId'))).compact(); + + // Update query param when the list of clients changes. + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpClient', serialize(intersection(clients, this.selectionClient))); + }); + + return clients.sort().map(dc => ({ key: dc, label: dc })); + } + + @action + setFacetQueryParam(queryParam, selection) { + this.set(queryParam, serialize(selection)); + } } diff --git a/ui/app/templates/jobs/job/task-group.hbs b/ui/app/templates/jobs/job/task-group.hbs index 2472c7b1b38e..87ef5d87a1ff 100644 --- a/ui/app/templates/jobs/job/task-group.hbs +++ b/ui/app/templates/jobs/job/task-group.hbs @@ -66,16 +66,32 @@
-
Allocations - +
+ + + +
{{#if this.sortedAllocations}} From d1ff1cb82f34c0da647acd5c38c5b173fd856a6a Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Tue, 23 Nov 2021 17:57:35 -0500 Subject: [PATCH 09/40] fix: filter callbacks use different param --- ui/app/controllers/jobs/job/allocations.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index d8e63f201df9..930bb85cc027 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -65,7 +65,7 @@ export default class AllocationsController extends Controller.extend( if (!allocations.length) return allocations; return allocations.filter(alloc => { - if (selectionStatus.length && !selectionStatus.includes(alloc.status)) { + if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { return false; } if (selectionClient.length && !selectionClient.includes(alloc.get('node.shortId'))) { @@ -113,7 +113,7 @@ export default class AllocationsController extends Controller.extend( this.set('qpClient', serialize(intersection(clients, this.selectionClient))); }); - return clients.sort().map(dc => ({ key: dc, label: dc })); + return clients.sort().map(c => ({ key: c, label: c })); } @computed('model.allocations.[]', 'selectionTaskGroup') @@ -126,7 +126,7 @@ export default class AllocationsController extends Controller.extend( this.set('qpTaskGroup', serialize(intersection(taskGroups, this.selectionTaskGroup))); }); - return taskGroups.sort().map(dc => ({ key: dc, label: dc })); + return taskGroups.sort().map(tg => ({ key: tg, label: tg })); } setFacetQueryParam(queryParam, selection) { From b394859aedb2868511298d81e5da101ceb46ad0b Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Tue, 23 Nov 2021 18:24:01 -0500 Subject: [PATCH 10/40] fix: add job version filter --- ui/app/controllers/jobs/job/allocations.js | 32 ++++++++++++++++++++-- ui/app/templates/jobs/job/allocations.hbs | 7 +++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index 930bb85cc027..7c5c289378e8 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -39,11 +39,15 @@ export default class AllocationsController extends Controller.extend( { qpTaskGroup: 'taskGroup', }, + { + qpJobVersion: 'jobVersion', + }, ]; qpStatus = ''; qpClient = ''; qpTaskGroup = ''; + qpJobVersion = ''; currentPage = 1; pageSize = 25; @@ -57,10 +61,16 @@ export default class AllocationsController extends Controller.extend( return ['shortId', 'name', 'taskGroupName']; } - @computed('model.allocations.[]', 'selectionStatus', 'selectionClient', 'selectionTaskGroup') + @computed( + 'model.allocations.[]', + 'selectionStatus', + 'selectionClient', + 'selectionTaskGroup', + 'selectionJobVersion' + ) get allocations() { const allocations = this.get('model.allocations') || []; - const { selectionStatus, selectionClient, selectionTaskGroup } = this; + const { selectionStatus, selectionClient, selectionTaskGroup, selectionJobVersion } = this; if (!allocations.length) return allocations; @@ -74,7 +84,9 @@ export default class AllocationsController extends Controller.extend( if (selectionTaskGroup.length && !selectionTaskGroup.includes(alloc.taskGroupName)) { return false; } - + if (selectionJobVersion.length && !selectionJobVersion.includes(alloc.jobVersion)) { + return false; + } return true; }); } @@ -82,6 +94,7 @@ export default class AllocationsController extends Controller.extend( @selection('qpStatus') selectionStatus; @selection('qpClient') selectionClient; @selection('qpTaskGroup') selectionTaskGroup; + @selection('qpJobVersion') selectionJobVersion; @alias('allocations') listToSort; @alias('listSorted') listToSearch; @@ -129,6 +142,19 @@ export default class AllocationsController extends Controller.extend( return taskGroups.sort().map(tg => ({ key: tg, label: tg })); } + @computed('model.allocations.[]', 'selectionJobVersion') + get optionsJobVersions() { + const jobVersions = Array.from(new Set(this.model.allocations.mapBy('jobVersion'))).compact(); + + // Update query param when the list of clients changes. + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpJobVersion', serialize(intersection(jobVersions, this.selectionJobVersion))); + }); + + return jobVersions.sort().map(jv => ({ key: jv, label: jv })); + } + setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); } diff --git a/ui/app/templates/jobs/job/allocations.hbs b/ui/app/templates/jobs/job/allocations.hbs index 2e164ad899f4..bcb76b60faa9 100644 --- a/ui/app/templates/jobs/job/allocations.hbs +++ b/ui/app/templates/jobs/job/allocations.hbs @@ -34,6 +34,13 @@ @selection={{this.selectionTaskGroup}} @onSelect={{action this.setFacetQueryParam "qpTaskGroup"}} /> +
From febcd5eafd69f36d57e9d582c95772057fd8d820 Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Tue, 23 Nov 2021 18:28:33 -0500 Subject: [PATCH 11/40] chore: changelog entry --- .changelog/11544.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/11544.txt diff --git a/.changelog/11544.txt b/.changelog/11544.txt new file mode 100644 index 000000000000..6c48985ccd8b --- /dev/null +++ b/.changelog/11544.txt @@ -0,0 +1,3 @@ +```release-note:feature +ui: feat: add filters to allocations table in jobs/job/allocation view +``` From 072d3b6b7407e0bba3ebba8e4b4550a5e30eced1 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Wed, 15 Dec 2021 10:44:03 -0500 Subject: [PATCH 12/40] cli: ensure `-stale` flag is respected by `nomad operator debug` (#11678) When a cluster doesn't have a leader, the `nomad operator debug` command can safely use stale queries to gracefully degrade the consistency of almost all its queries. The query parameter for these API calls was not being set by the command. Some `api` package queries do not include `QueryOptions` because they target a specific agent, but they can potentially be forwarded to other agents. If there is no leader, these forwarded queries will fail. Provide methods to call these APIs with `QueryOptions`. --- .changelog/11678.txt | 3 + api/agent.go | 11 ++++ api/nodes.go | 9 +++ api/regions.go | 3 +- command/agent/agent_endpoint.go | 8 ++- command/operator_debug.go | 104 ++++++++++++++++++++------------ command/operator_debug_test.go | 51 ++++++++++++++++ 7 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 .changelog/11678.txt diff --git a/.changelog/11678.txt b/.changelog/11678.txt new file mode 100644 index 000000000000..c82272549fb4 --- /dev/null +++ b/.changelog/11678.txt @@ -0,0 +1,3 @@ +```release-note:bug +cli: Fixed a bug where the `-stale` flag was not respected by `nomad operator debug` +``` diff --git a/api/agent.go b/api/agent.go index 39bfb95443cd..424e9ad95d28 100644 --- a/api/agent.go +++ b/api/agent.go @@ -147,6 +147,17 @@ func (a *Agent) Members() (*ServerMembers, error) { return resp, nil } +// Members is used to query all of the known server members +// with the ability to set QueryOptions +func (a *Agent) MembersOpts(opts *QueryOptions) (*ServerMembers, error) { + var resp *ServerMembers + _, err := a.client.query("/v1/agent/members", &resp, opts) + if err != nil { + return nil, err + } + return resp, nil +} + // ForceLeave is used to eject an existing node from the cluster. func (a *Agent) ForceLeave(node string) error { _, err := a.client.write("/v1/agent/force-leave?node="+node, nil, nil, nil) diff --git a/api/nodes.go b/api/nodes.go index 488f5eb625df..2c86651dbb9e 100644 --- a/api/nodes.go +++ b/api/nodes.go @@ -49,6 +49,15 @@ func (n *Nodes) PrefixList(prefix string) ([]*NodeListStub, *QueryMeta, error) { return n.List(&QueryOptions{Prefix: prefix}) } +func (n *Nodes) PrefixListOpts(prefix string, opts *QueryOptions) ([]*NodeListStub, *QueryMeta, error) { + if opts == nil { + opts = &QueryOptions{Prefix: prefix} + } else { + opts.Prefix = prefix + } + return n.List(opts) +} + // Info is used to query a specific node by its ID. func (n *Nodes) Info(nodeID string, q *QueryOptions) (*Node, *QueryMeta, error) { var resp Node diff --git a/api/regions.go b/api/regions.go index c94ce297a892..98df011d04e0 100644 --- a/api/regions.go +++ b/api/regions.go @@ -12,7 +12,8 @@ func (c *Client) Regions() *Regions { return &Regions{client: c} } -// List returns a list of all of the regions. +// List returns a list of all of the regions from the server +// that serves the request. It is never forwarded to a leader. func (r *Regions) List() ([]string, error) { var resp []string if _, err := r.client.query("/v1/regions", &resp, nil); err != nil { diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go index dc4afd1ef6c7..798d65487f80 100644 --- a/command/agent/agent_endpoint.go +++ b/command/agent/agent_endpoint.go @@ -71,7 +71,9 @@ func (s *HTTPServer) AgentSelfRequest(resp http.ResponseWriter, req *http.Reques member = srv.LocalMember() aclObj, err = srv.ResolveToken(secret) } else { - // Not a Server; use the Client for token resolution + // Not a Server, so use the Client for token resolution. Note + // this gets forwarded to a server with AllowStale = true if + // the local ACL cache TTL has expired (30s by default) aclObj, err = s.agent.Client().ResolveToken(secret) } @@ -677,7 +679,9 @@ func (s *HTTPServer) AgentHostRequest(resp http.ResponseWriter, req *http.Reques aclObj, err = srv.ResolveToken(secret) enableDebug = srv.GetConfig().EnableDebug } else { - // Not a Server; use the Client for token resolution + // Not a Server, so use the Client for token resolution. Note + // this gets forwarded to a server with AllowStale = true if + // the local ACL cache TTL has expired (30s by default) aclObj, err = s.agent.Client().ResolveToken(secret) enableDebug = s.agent.Client().GetConfig().EnableDebug } diff --git a/command/operator_debug.go b/command/operator_debug.go index 245be9086f1b..21b503c7906d 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -38,7 +38,6 @@ type OperatorDebugCommand struct { interval time.Duration pprofDuration time.Duration logLevel string - stale bool maxNodes int nodeClass string nodeIDs []string @@ -48,6 +47,7 @@ type OperatorDebugCommand struct { manifest []string ctx context.Context cancel context.CancelFunc + opts *api.QueryOptions } const ( @@ -140,7 +140,7 @@ Debug Options: nodes at "log-level". Defaults to 2m. -interval= - The interval between snapshots of the Nomad state. Set interval equal to + The interval between snapshots of the Nomad state. Set interval equal to duration to capture a single snapshot. Defaults to 30s. -log-level= @@ -172,7 +172,7 @@ Debug Options: necessary to get the configuration from a non-leader server. -output= - Path to the parent directory of the output directory. If specified, no + Path to the parent directory of the output directory. If specified, no archive is built. Defaults to the current directory. ` return strings.TrimSpace(helpText) @@ -211,7 +211,12 @@ func NodePredictor(factory ApiClientFactory) complete.Predictor { return nil } - resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil) + // note we can't use the -stale flag here because we're in the + // predictor, but a stale query should be safe for prediction; + // we also can't use region forwarding because we can't rely + // on the server being up + resp, _, err := client.Search().PrefixSearch( + a.Last, contexts.Nodes, &api.QueryOptions{AllowStale: true}) if err != nil { return []string{} } @@ -228,7 +233,11 @@ func NodeClassPredictor(factory ApiClientFactory) complete.Predictor { return nil } - nodes, _, err := client.Nodes().List(nil) // TODO: should be *api.QueryOptions that matches region + // note we can't use the -stale flag here because we're in the + // predictor, but a stale query should be safe for prediction; + // we also can't use region forwarding because we can't rely + // on the server being up + nodes, _, err := client.Nodes().List(&api.QueryOptions{AllowStale: true}) if err != nil { return []string{} } @@ -259,7 +268,12 @@ func ServerPredictor(factory ApiClientFactory) complete.Predictor { if err != nil { return nil } - members, err := client.Agent().Members() + + // note we can't use the -stale flag here because we're in the + // predictor, but a stale query should be safe for prediction; + // we also can't use region forwarding because we can't rely + // on the server being up + members, err := client.Agent().MembersOpts(&api.QueryOptions{AllowStale: true}) if err != nil { return []string{} } @@ -276,6 +290,15 @@ func ServerPredictor(factory ApiClientFactory) complete.Predictor { }) } +// queryOpts returns a copy of the shared api.QueryOptions so +// that api package methods can safely modify the options +func (c *OperatorDebugCommand) queryOpts() *api.QueryOptions { + qo := new(api.QueryOptions) + *qo = *c.opts + qo.Params = helper.CopyMapStringString(c.opts.Params) + return qo +} + func (c *OperatorDebugCommand) Name() string { return "debug" } func (c *OperatorDebugCommand) Run(args []string) int { @@ -284,6 +307,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { var duration, interval, output, pprofDuration string var nodeIDs, serverIDs string + var allowStale bool flags.StringVar(&duration, "duration", "2m", "") flags.StringVar(&interval, "interval", "30s", "") @@ -292,7 +316,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { flags.StringVar(&c.nodeClass, "node-class", "", "") flags.StringVar(&nodeIDs, "node-id", "all", "") flags.StringVar(&serverIDs, "server-id", "all", "") - flags.BoolVar(&c.stale, "stale", false, "") + flags.BoolVar(&allowStale, "stale", false, "") flags.StringVar(&output, "output", "", "") flags.StringVar(&pprofDuration, "pprof-duration", "1s", "") @@ -403,6 +427,12 @@ func (c *OperatorDebugCommand) Run(args []string) int { return 1 } + c.opts = &api.QueryOptions{ + Region: c.Meta.region, + AllowStale: allowStale, + AuthToken: c.Meta.token, + } + // Search all nodes If a node class is specified without a list of node id prefixes if c.nodeClass != "" && nodeIDs == "" { nodeIDs = "all" @@ -421,7 +451,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { // Capture from nodes starting with prefix id id = sanitizeUUIDPrefix(id) } - nodes, _, err := client.Nodes().PrefixList(id) + nodes, _, err := client.Nodes().PrefixListOpts(id, c.queryOpts()) if err != nil { c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) return 1 @@ -466,7 +496,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { } // Resolve servers - members, err := client.Agent().Members() + members, err := client.Agent().MembersOpts(c.queryOpts()) if err != nil { c.Ui.Error(fmt.Sprintf("Failed to retrieve server list; err: %v", err)) return 1 @@ -559,8 +589,7 @@ func (c *OperatorDebugCommand) collect(client *api.Client) error { self, err := client.Agent().Self() c.writeJSON(clusterDir, "agent-self.json", self, err) - var qo *api.QueryOptions - namespaces, _, err := client.Namespaces().List(qo) + namespaces, _, err := client.Namespaces().List(c.queryOpts()) c.writeJSON(clusterDir, "namespaces.json", namespaces, err) regions, err := client.Regions().List() @@ -635,6 +664,7 @@ func (c *OperatorDebugCommand) startMonitor(path, idKey, nodeID string, client * idKey: nodeID, "log_level": c.logLevel, }, + AllowStale: c.queryOpts().AllowStale, } outCh, errCh := client.Agent().Monitor(c.ctx.Done(), &qo) @@ -672,9 +702,9 @@ func (c *OperatorDebugCommand) collectAgentHost(path, id string, client *api.Cli var host *api.HostDataResponse var err error if path == serverDir { - host, err = client.Agent().Host(id, "", nil) + host, err = client.Agent().Host(id, "", c.queryOpts()) } else { - host, err = client.Agent().Host("", id, nil) + host, err = client.Agent().Host("", id, c.queryOpts()) } if err != nil { @@ -714,7 +744,7 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client) path = filepath.Join(path, id) - bs, err := client.Agent().CPUProfile(opts, nil) + bs, err := client.Agent().CPUProfile(opts, c.queryOpts()) if err != nil { c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof profile.prof, err: %v", path, err)) if structs.IsErrPermissionDenied(err) { @@ -763,7 +793,7 @@ func (c *OperatorDebugCommand) savePprofProfile(path string, profile string, opt fileName = fmt.Sprintf("%s-debug%d.txt", profile, opts.Debug) } - bs, err := retrievePprofProfile(profile, opts, client) + bs, err := retrievePprofProfile(profile, opts, client, c.queryOpts()) if err != nil { c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof %s, err: %s", path, fileName, err.Error())) } @@ -774,22 +804,23 @@ func (c *OperatorDebugCommand) savePprofProfile(path string, profile string, opt } } -// retrievePprofProfile gets a pprof profile from the node specified in opts using the API client -func retrievePprofProfile(profile string, opts api.PprofOptions, client *api.Client) (bs []byte, err error) { +// retrievePprofProfile gets a pprof profile from the node specified +// in opts using the API client +func retrievePprofProfile(profile string, opts api.PprofOptions, client *api.Client, qopts *api.QueryOptions) (bs []byte, err error) { switch profile { case "cpuprofile": - bs, err = client.Agent().CPUProfile(opts, nil) + bs, err = client.Agent().CPUProfile(opts, qopts) case "trace": - bs, err = client.Agent().Trace(opts, nil) + bs, err = client.Agent().Trace(opts, qopts) default: - bs, err = client.Agent().Lookup(profile, opts, nil) + bs, err = client.Agent().Lookup(profile, opts, qopts) } return bs, err } -// collectPeriodic runs for duration, capturing the cluster state every interval. It flushes and stops -// the monitor requests +// collectPeriodic runs for duration, capturing the cluster state +// every interval. It flushes and stops the monitor requests func (c *OperatorDebugCommand) collectPeriodic(client *api.Client) { duration := time.After(c.duration) // Set interval to 0 so that we immediately execute, wait the interval next time @@ -820,61 +851,60 @@ func (c *OperatorDebugCommand) collectPeriodic(client *api.Client) { // collectOperator captures some cluster meta information func (c *OperatorDebugCommand) collectOperator(dir string, client *api.Client) { - rc, err := client.Operator().RaftGetConfiguration(nil) + rc, err := client.Operator().RaftGetConfiguration(c.queryOpts()) c.writeJSON(dir, "operator-raft.json", rc, err) - sc, _, err := client.Operator().SchedulerGetConfiguration(nil) + sc, _, err := client.Operator().SchedulerGetConfiguration(c.queryOpts()) c.writeJSON(dir, "operator-scheduler.json", sc, err) - ah, _, err := client.Operator().AutopilotServerHealth(nil) + ah, _, err := client.Operator().AutopilotServerHealth(c.queryOpts()) c.writeJSON(dir, "operator-autopilot-health.json", ah, err) - lic, _, err := client.Operator().LicenseGet(nil) + lic, _, err := client.Operator().LicenseGet(c.queryOpts()) c.writeJSON(dir, "license.json", lic, err) } // collectNomad captures the nomad cluster state func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) error { - var qo *api.QueryOptions - js, _, err := client.Jobs().List(qo) + js, _, err := client.Jobs().List(c.queryOpts()) c.writeJSON(dir, "jobs.json", js, err) - ds, _, err := client.Deployments().List(qo) + ds, _, err := client.Deployments().List(c.queryOpts()) c.writeJSON(dir, "deployments.json", ds, err) - es, _, err := client.Evaluations().List(qo) + es, _, err := client.Evaluations().List(c.queryOpts()) c.writeJSON(dir, "evaluations.json", es, err) - as, _, err := client.Allocations().List(qo) + as, _, err := client.Allocations().List(c.queryOpts()) c.writeJSON(dir, "allocations.json", as, err) - ns, _, err := client.Nodes().List(qo) + ns, _, err := client.Nodes().List(c.queryOpts()) c.writeJSON(dir, "nodes.json", ns, err) // CSI Plugins - /v1/plugins?type=csi - ps, _, err := client.CSIPlugins().List(qo) + ps, _, err := client.CSIPlugins().List(c.queryOpts()) c.writeJSON(dir, "csi-plugins.json", ps, err) // CSI Plugin details - /v1/plugin/csi/:plugin_id for _, p := range ps { - csiPlugin, _, err := client.CSIPlugins().Info(p.ID, qo) + csiPlugin, _, err := client.CSIPlugins().Info(p.ID, c.queryOpts()) csiPluginFileName := fmt.Sprintf("csi-plugin-id-%s.json", p.ID) c.writeJSON(dir, csiPluginFileName, csiPlugin, err) } // CSI Volumes - /v1/volumes?type=csi - csiVolumes, _, err := client.CSIVolumes().List(qo) + csiVolumes, _, err := client.CSIVolumes().List(c.queryOpts()) c.writeJSON(dir, "csi-volumes.json", csiVolumes, err) // CSI Volume details - /v1/volumes/csi/:volume-id for _, v := range csiVolumes { - csiVolume, _, err := client.CSIVolumes().Info(v.ID, qo) + csiVolume, _, err := client.CSIVolumes().Info(v.ID, c.queryOpts()) csiFileName := fmt.Sprintf("csi-volume-id-%s.json", v.ID) c.writeJSON(dir, csiFileName, csiVolume, err) } - metrics, _, err := client.Operator().MetricsSummary(qo) + metrics, _, err := client.Operator().MetricsSummary(c.queryOpts()) c.writeJSON(dir, "metrics.json", metrics, err) return nil diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index c61419faf15c..50f3a8756eee 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -740,3 +740,54 @@ func TestDebug_CollectVault(t *testing.T) { require.FileExists(t, filepath.Join(testDir, "test", "vault-sys-health.json")) } + +// TestDebug_StaleLeadership verifies that APIs that are required to +// complete a debug run have their query options configured with the +// -stale flag +func TestDebug_StaleLeadership(t *testing.T) { + srv, _, url := testServerWithoutLeader(t, false, nil) + addrServer := srv.HTTPAddr() + + t.Logf("[TEST] testAgent api address: %s", url) + t.Logf("[TEST] Server api address: %s", addrServer) + + var cases = testCases{ + { + name: "no leader without stale flag", + args: []string{"-address", addrServer, + "-duration", "250ms", "-interval", "250ms", + "-server-id", "all", "-node-id", "all"}, + expectedCode: 1, + }, + { + name: "no leader with stale flag", + args: []string{ + "-address", addrServer, + "-duration", "250ms", "-interval", "250ms", + "-server-id", "all", "-node-id", "all", + "-stale"}, + expectedCode: 0, + expectedOutputs: []string{"Created debug archive"}, + }, + } + + runTestCases(t, cases) +} + +func testServerWithoutLeader(t *testing.T, runClient bool, cb func(*agent.Config)) (*agent.TestAgent, *api.Client, string) { + // Make a new test server + a := agent.NewTestAgent(t, t.Name(), func(config *agent.Config) { + config.Client.Enabled = runClient + config.Server.Enabled = true + config.Server.NumSchedulers = helper.IntToPtr(0) + config.Server.BootstrapExpect = 3 + + if cb != nil { + cb(config) + } + }) + t.Cleanup(func() { a.Shutdown() }) + + c := a.Client() + return a, c, a.HTTPAddr() +} From 97621ec3c5fdd32a331e6e7f14fd4fc6f2257212 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Wed, 15 Dec 2021 11:58:38 -0500 Subject: [PATCH 13/40] `nomad eval list` command (#11675) Use the new filtering and pagination capabilities of the `Eval.List` RPC to provide filtering and pagination at the command line. Also includes note that `nomad eval status -json` is deprecated and will be replaced with a single evaluation view in a future version of Nomad. --- .changelog/11675.txt | 3 + command/commands.go | 5 + command/eval_list.go | 213 ++++++++++++++++++ command/eval_list_test.go | 60 +++++ website/content/docs/commands/eval/index.mdx | 23 ++ website/content/docs/commands/eval/list.mdx | 51 +++++ .../{eval-status.mdx => eval/status.mdx} | 5 +- .../content/docs/upgrade/upgrade-specific.mdx | 10 + website/data/docs-nav-data.json | 17 +- website/redirects.js | 5 + 10 files changed, 389 insertions(+), 3 deletions(-) create mode 100644 .changelog/11675.txt create mode 100644 command/eval_list.go create mode 100644 command/eval_list_test.go create mode 100644 website/content/docs/commands/eval/index.mdx create mode 100644 website/content/docs/commands/eval/list.mdx rename website/content/docs/commands/{eval-status.mdx => eval/status.mdx} (91%) diff --git a/.changelog/11675.txt b/.changelog/11675.txt new file mode 100644 index 000000000000..d5f12c50082c --- /dev/null +++ b/.changelog/11675.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: Added a `nomad eval list` command. +``` diff --git a/command/commands.go b/command/commands.go index 87f46b8f4ad0..9b58cdd8f914 100644 --- a/command/commands.go +++ b/command/commands.go @@ -255,6 +255,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "eval list": func() (cli.Command, error) { + return &EvalListCommand{ + Meta: meta, + }, nil + }, "eval status": func() (cli.Command, error) { return &EvalStatusCommand{ Meta: meta, diff --git a/command/eval_list.go b/command/eval_list.go new file mode 100644 index 000000000000..20ea04477236 --- /dev/null +++ b/command/eval_list.go @@ -0,0 +1,213 @@ +package command + +import ( + "fmt" + "os" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/api/contexts" + "github.com/posener/complete" +) + +type EvalListCommand struct { + Meta +} + +func (c *EvalListCommand) Help() string { + helpText := ` +Usage: nomad eval list [options] + + List is used to list the set of evaluations processed by Nomad. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Eval List Options: + + -verbose + Show full information. + + -per-page + How many results to show per page. + + -page-token + Where to start pagination. + + -job + Only show evaluations for this job ID. + + -status + Only show evaluations with this status. + + -json + Output the evaluation in its JSON format. + + -t + Format and display evaluation using a Go template. +` + + return strings.TrimSpace(helpText) +} + +func (c *EvalListCommand) Synopsis() string { + return "List the set of evaluations processed by Nomad" +} + +func (c *EvalListCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-json": complete.PredictNothing, + "-t": complete.PredictAnything, + "-verbose": complete.PredictNothing, + "-job": complete.PredictAnything, + "-status": complete.PredictAnything, + "-per-page": complete.PredictAnything, + "-page-token": complete.PredictAnything, + }) +} + +func (c *EvalListCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictFunc(func(a complete.Args) []string { + client, err := c.Meta.Client() + if err != nil { + return nil + } + + resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Evals, nil) + if err != nil { + return []string{} + } + return resp.Matches[contexts.Evals] + }) +} + +func (c *EvalListCommand) Name() string { return "eval list" } + +func (c *EvalListCommand) Run(args []string) int { + var monitor, verbose, json bool + var perPage int + var tmpl, pageToken, filterJobID, filterStatus string + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&monitor, "monitor", false, "") + flags.BoolVar(&verbose, "verbose", false, "") + flags.BoolVar(&json, "json", false, "") + flags.StringVar(&tmpl, "t", "", "") + flags.IntVar(&perPage, "per-page", 0, "") + flags.StringVar(&pageToken, "page-token", "", "") + flags.StringVar(&filterJobID, "job", "", "") + flags.StringVar(&filterStatus, "status", "", "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got no arguments + args = flags.Args() + if l := len(args); l != 0 { + c.Ui.Error("This command takes no arguments") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + opts := &api.QueryOptions{ + PerPage: int32(perPage), + NextToken: pageToken, + Params: map[string]string{}, + } + if filterJobID != "" { + opts.Params["job"] = filterJobID + } + if filterStatus != "" { + opts.Params["status"] = filterStatus + } + + evals, qm, err := client.Evaluations().List(opts) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying evaluations: %v", err)) + return 1 + } + + // If args not specified but output format is specified, format + // and output the evaluations data list + if json || len(tmpl) > 0 { + out, err := Format(json, tmpl, evals) + if err != nil { + c.Ui.Error(err.Error()) + return 1 + } + + c.Ui.Output(out) + return 0 + } + + if len(evals) == 0 { + c.Ui.Output("No evals found") + return 0 + } + + // Truncate the id unless full length is requested + length := shortId + if verbose { + length = fullId + } + + out := make([]string, len(evals)+1) + out[0] = "ID|Priority|Triggered By|Job ID|Status|Placement Failures" + for i, eval := range evals { + failures, _ := evalFailureStatus(eval) + out[i+1] = fmt.Sprintf("%s|%d|%s|%s|%s|%s", + limit(eval.ID, length), + eval.Priority, + eval.TriggeredBy, + eval.JobID, + eval.Status, + failures, + ) + } + c.Ui.Output(formatList(out)) + + if qm.NextToken != "" { + c.Ui.Output(fmt.Sprintf(` +Results have been paginated. To get the next page run: + +%s -page-token %s`, argsWithoutPageToken(os.Args), qm.NextToken)) + } + + return 0 +} + +// argsWithoutPageToken strips out of the -page-token argument and +// returns the joined string +func argsWithoutPageToken(osArgs []string) string { + args := []string{} + i := 0 + for { + if i >= len(osArgs) { + break + } + arg := osArgs[i] + + if strings.HasPrefix(arg, "-page-token") { + if strings.Contains(arg, "=") { + i += 1 + } else { + i += 2 + } + continue + } + + args = append(args, arg) + i++ + } + return strings.Join(args, " ") +} diff --git a/command/eval_list_test.go b/command/eval_list_test.go new file mode 100644 index 000000000000..0984b3ac88ca --- /dev/null +++ b/command/eval_list_test.go @@ -0,0 +1,60 @@ +package command + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEvalList_ArgsWithoutPageToken(t *testing.T) { + + cases := []struct { + cli string + expected string + }{ + { + cli: "nomad eval list -page-token=abcdef", + expected: "nomad eval list", + }, + { + cli: "nomad eval list -page-token abcdef", + expected: "nomad eval list", + }, + { + cli: "nomad eval list -per-page 3 -page-token abcdef", + expected: "nomad eval list -per-page 3", + }, + { + cli: "nomad eval list -page-token abcdef -per-page 3", + expected: "nomad eval list -per-page 3", + }, + { + cli: "nomad eval list -per-page=3 -page-token abcdef", + expected: "nomad eval list -per-page=3", + }, + { + cli: "nomad eval list -verbose -page-token abcdef", + expected: "nomad eval list -verbose", + }, + { + cli: "nomad eval list -page-token abcdef -verbose", + expected: "nomad eval list -verbose", + }, + { + cli: "nomad eval list -verbose -page-token abcdef -per-page 3", + expected: "nomad eval list -verbose -per-page 3", + }, + { + cli: "nomad eval list -page-token abcdef -verbose -per-page 3", + expected: "nomad eval list -verbose -per-page 3", + }, + } + + for _, tc := range cases { + args := strings.Split(tc.cli, " ") + assert.Equal(t, tc.expected, argsWithoutPageToken(args), + "for input: %s", tc.cli) + } + +} diff --git a/website/content/docs/commands/eval/index.mdx b/website/content/docs/commands/eval/index.mdx new file mode 100644 index 000000000000..504cd8a17772 --- /dev/null +++ b/website/content/docs/commands/eval/index.mdx @@ -0,0 +1,23 @@ +--- +layout: docs +page_title: 'Commands: eval' +description: | + The eval command is used to interact with evals. +--- + +# Command: eval + +The `eval` command is used to interact with evals. + +## Usage + +Usage: `nomad eval [options]` + +Run `nomad eval -h` for help on that subcommand. The following +subcommands are available: + +- [`eval list`][list] - List all evals +- [`eval status`][status] - Display the status of a eval + +[list]: /docs/commands/eval/list 'List all evals' +[status]: /docs/commands/eval/status 'Display the status of a eval' diff --git a/website/content/docs/commands/eval/list.mdx b/website/content/docs/commands/eval/list.mdx new file mode 100644 index 000000000000..467fbb90b9c4 --- /dev/null +++ b/website/content/docs/commands/eval/list.mdx @@ -0,0 +1,51 @@ +--- +layout: docs +page_title: 'Commands: eval list' +description: | + The eval list command is used to list evaluations. +--- + +# Command: eval list + +The `eval list` command is used list all evaluations. + +## Usage + +```plaintext +nomad eval list [options] +``` + +The `eval list` command requires no arguments. + +When ACLs are enabled, this command requires a token with the `read-job` +capability for the requested namespace. + +## General Options + +@include 'general_options.mdx' + +## List Options + +- `-verbose`: Show full information. +- `-per-page`: How many results to show per page. +- `-page-token`: Where to start pagination. +- `-job`: Only show evaluations for this job ID. +- `-status`: Only show evaluations with this status. +- `-json`: Output the evaluation in its JSON format. +- `-t`: Format and display evaluation using a Go template. + +## Examples + +List all tracked evaluations: + +```shell-session +$ nomad eval list -per-page 3 -status complete +ID Priority Triggered By Job ID Status Placement Failures +456e37aa 50 deployment-watcher example complete false +1a1eafe6 50 alloc-stop example complete false +3411e37b 50 job-register example complete false + +Results have been paginated. To get the next page run: + +nomad eval list -page-token 9ecffbba-73be-d909-5d7e-ac2694c10e0c +``` diff --git a/website/content/docs/commands/eval-status.mdx b/website/content/docs/commands/eval/status.mdx similarity index 91% rename from website/content/docs/commands/eval-status.mdx rename to website/content/docs/commands/eval/status.mdx index 6d93efc4693d..f89bb603b050 100644 --- a/website/content/docs/commands/eval-status.mdx +++ b/website/content/docs/commands/eval/status.mdx @@ -45,7 +45,10 @@ indicated by exit code 1. - `-monitor`: Monitor an outstanding evaluation - `-verbose`: Show full information. -- `-json` : Output the evaluation in its JSON format. +- `-json` : Output a list of all evaluations in JSON format. This + behavior is deprecated and has been replaced by `nomad eval list + -json`. In Nomad 1.4.0 the behavior of this option will change to + output only the selected evaluation in JSON. - `-t` : Format and display evaluation using a Go template. ## Examples diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 1035bd56d347..90b5a95a2b92 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -13,6 +13,16 @@ upgrade. However, specific versions of Nomad may have more details provided for their upgrades as a result of new features or changed behavior. This page is used to document those details separately from the standard upgrade flow. +## Nomad 1.2.4 + +#### `nomad eval status -json` deprecated + +Nomad 1.2.4 includes a new `nomad eval list` command that has the +option to display the results in JSON format with the `-json` +flag. This replaces the existing `nomad eval status -json` option. In +Nomad 1.4.0, `nomad eval status -json` will be changed to display only +the selected evaluation in JSON format. + ## Nomad 1.2.2 ### Panic on node class filtering for system and sysbatch jobs fixed diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 51ce0a2af9eb..28f67624c5fb 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -353,8 +353,21 @@ ] }, { - "title": "eval status", - "path": "commands/eval-status" + "title": "eval", + "routes": [ + { + "title": "Overview", + "path": "commands/eval" + }, + { + "title": "list", + "path": "commands/eval/list" + }, + { + "title": "status", + "path": "commands/eval/status" + } + ] }, { "title": "job", diff --git a/website/redirects.js b/website/redirects.js index 48a206b7f66c..0ed8488b8b2d 100644 --- a/website/redirects.js +++ b/website/redirects.js @@ -589,6 +589,11 @@ module.exports = [ destination: '/docs/commands/alloc/status', permanent: true, }, + { + source: '/docs/commands/eval-status', + destination: '/docs/commands/eval/status', + permanent: true, + }, { source: '/docs/commands/fs', destination: '/docs/commands/alloc/fs', From 03ea7d1c1795d878dc233c07834741026295e693 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 16 Dec 2021 10:32:11 -0500 Subject: [PATCH 14/40] cli: unhide advanced operator raft debugging commands (#11682) The `nomad operator raft` and `nomad operator snapshot state` subcommands for inspecting on-disk raft state were hidden and undocumented. Expose and document these so that advanced operators have support for these tools. --- .changelog/11682.txt | 3 ++ command/commands.go | 8 ++-- command/operator_raft.go | 15 ++++++ command/operator_raft_info.go | 12 +++-- command/operator_raft_logs.go | 12 +++-- command/operator_raft_state.go | 13 ++++-- command/operator_snapshot_state.go | 6 +-- .../docs/commands/operator/raft-info.mdx | 39 ++++++++++++++++ .../commands/operator/raft-list-peers.mdx | 2 +- .../docs/commands/operator/raft-logs.mdx | 38 +++++++++++++++ .../docs/commands/operator/raft-state.mdx | 46 +++++++++++++++++++ .../docs/commands/operator/snapshot-state.mdx | 30 ++++++++++++ website/data/docs-nav-data.json | 16 +++++++ 13 files changed, 218 insertions(+), 22 deletions(-) create mode 100644 .changelog/11682.txt create mode 100644 website/content/docs/commands/operator/raft-info.mdx create mode 100644 website/content/docs/commands/operator/raft-logs.mdx create mode 100644 website/content/docs/commands/operator/raft-state.mdx create mode 100644 website/content/docs/commands/operator/snapshot-state.mdx diff --git a/.changelog/11682.txt b/.changelog/11682.txt new file mode 100644 index 000000000000..f667fec3e14e --- /dev/null +++ b/.changelog/11682.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: Made the `operator raft info`, `operator raft logs`, `operator raft state`, and `operator snapshot state` commands visible to command line help. +``` diff --git a/command/commands.go b/command/commands.go index 9b58cdd8f914..52075364e8bf 100644 --- a/command/commands.go +++ b/command/commands.go @@ -540,17 +540,17 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "operator raft _info": func() (cli.Command, error) { + "operator raft info": func() (cli.Command, error) { return &OperatorRaftInfoCommand{ Meta: meta, }, nil }, - "operator raft _logs": func() (cli.Command, error) { + "operator raft logs": func() (cli.Command, error) { return &OperatorRaftLogsCommand{ Meta: meta, }, nil }, - "operator raft _state": func() (cli.Command, error) { + "operator raft state": func() (cli.Command, error) { return &OperatorRaftStateCommand{ Meta: meta, }, nil @@ -571,7 +571,7 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "operator snapshot _state": func() (cli.Command, error) { + "operator snapshot state": func() (cli.Command, error) { return &OperatorSnapshotStateCommand{ Meta: meta, }, nil diff --git a/command/operator_raft.go b/command/operator_raft.go index 79e0cb6482ca..a1aac48755e6 100644 --- a/command/operator_raft.go +++ b/command/operator_raft.go @@ -26,7 +26,22 @@ Usage: nomad operator raft [options] $ nomad operator raft remove-peer -peer-address "IP:Port" + Display info about the raft logs in the data directory: + + $ nomad operator raft info /var/nomad/data + + Display the log entries persisted in data dir in JSON format. + + $ nomad operator raft logs /var/nomad/data + + Display the server state obtained by replaying raft log entries + persisted in data dir in JSON format. + + $ nomad operator raft state /var/nomad/data + Please see the individual subcommand help for detailed usage information. + + ` return strings.TrimSpace(helpText) } diff --git a/command/operator_raft_info.go b/command/operator_raft_info.go index 8afe730237fa..18622e81044c 100644 --- a/command/operator_raft_info.go +++ b/command/operator_raft_info.go @@ -14,14 +14,16 @@ type OperatorRaftInfoCommand struct { func (c *OperatorRaftInfoCommand) Help() string { helpText := ` -Usage: nomad operator raft _info +Usage: nomad operator raft info - Displays info about the raft logs in the data directory. + Displays summary information about the raft logs in the data directory. + + This command requires file system permissions to access the data directory on + disk. The Nomad server locks access to the data directory, so this command + cannot be run on a data directory that is being used by a running Nomad server. This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. - - If ACLs are enabled, this command requires a management token. ` return strings.TrimSpace(helpText) } @@ -38,7 +40,7 @@ func (c *OperatorRaftInfoCommand) Synopsis() string { return "Display info of the raft log" } -func (c *OperatorRaftInfoCommand) Name() string { return "operator raft _info" } +func (c *OperatorRaftInfoCommand) Name() string { return "operator raft info" } func (c *OperatorRaftInfoCommand) Run(args []string) int { if len(args) != 1 { diff --git a/command/operator_raft_logs.go b/command/operator_raft_logs.go index e85649c4ea10..fe3233ee6c9c 100644 --- a/command/operator_raft_logs.go +++ b/command/operator_raft_logs.go @@ -16,14 +16,18 @@ type OperatorRaftLogsCommand struct { func (c *OperatorRaftLogsCommand) Help() string { helpText := ` -Usage: nomad operator raft _logs +Usage: nomad operator raft logs - Display the log entries persisted in data dir in json form. + Display the log entries persisted in the Nomad data directory in JSON + format. + + This command requires file system permissions to access the data directory on + disk. The Nomad server locks access to the data directory, so this command + cannot be run on a data directory that is being used by a running Nomad server. This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. - If ACLs are enabled, this command requires a management token. ` return strings.TrimSpace(helpText) } @@ -40,7 +44,7 @@ func (c *OperatorRaftLogsCommand) Synopsis() string { return "Display raft log content" } -func (c *OperatorRaftLogsCommand) Name() string { return "operator raft _info" } +func (c *OperatorRaftLogsCommand) Name() string { return "operator raft logs" } func (c *OperatorRaftLogsCommand) Run(args []string) int { if len(args) != 1 { diff --git a/command/operator_raft_state.go b/command/operator_raft_state.go index 2b70a329f394..a116ae97f1f2 100644 --- a/command/operator_raft_state.go +++ b/command/operator_raft_state.go @@ -16,15 +16,18 @@ type OperatorRaftStateCommand struct { func (c *OperatorRaftStateCommand) Help() string { helpText := ` -Usage: nomad operator raft _state +Usage: nomad operator raft state - Display the server state obtained by replaying raft log entries persisted in data dir in json form. + Display the server state obtained by replaying raft log entries persisted in + the Nomad data directory in JSON format. + + This command requires file system permissions to access the data directory on + disk. The Nomad server locks access to the data directory, so this command + cannot be run on a data directory that is being used by a running Nomad server. This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. - If ACLs are enabled, this command requires a management token. - Options: -last-index= @@ -47,7 +50,7 @@ func (c *OperatorRaftStateCommand) Synopsis() string { return "Display raft server state" } -func (c *OperatorRaftStateCommand) Name() string { return "operator raft _state" } +func (c *OperatorRaftStateCommand) Name() string { return "operator raft state" } func (c *OperatorRaftStateCommand) Run(args []string) int { var fLastIdx int64 diff --git a/command/operator_snapshot_state.go b/command/operator_snapshot_state.go index ed7b66346d5e..5ba28a56c968 100644 --- a/command/operator_snapshot_state.go +++ b/command/operator_snapshot_state.go @@ -16,13 +16,13 @@ type OperatorSnapshotStateCommand struct { func (c *OperatorSnapshotStateCommand) Help() string { helpText := ` -Usage: nomad operator snapshot _state +Usage: nomad operator snapshot state Displays a JSON representation of state in the snapshot. To inspect the file "backup.snap": - $ nomad operator snapshot _state backup.snap + $ nomad operator snapshot state backup.snap ` return strings.TrimSpace(helpText) } @@ -39,7 +39,7 @@ func (c *OperatorSnapshotStateCommand) Synopsis() string { return "Displays information about a Nomad snapshot file" } -func (c *OperatorSnapshotStateCommand) Name() string { return "operator snapshot _state" } +func (c *OperatorSnapshotStateCommand) Name() string { return "operator snapshot state" } func (c *OperatorSnapshotStateCommand) Run(args []string) int { // Check that we either got no filename or exactly one. diff --git a/website/content/docs/commands/operator/raft-info.mdx b/website/content/docs/commands/operator/raft-info.mdx new file mode 100644 index 000000000000..fc17543468fb --- /dev/null +++ b/website/content/docs/commands/operator/raft-info.mdx @@ -0,0 +1,39 @@ +--- +layout: docs +page_title: 'Commands: operator raft info' +description: | + Display Raft server state. +--- + +# Command: operator raft info + +The `raft info` command is used to display summary information about the +raft logs persisted in the Nomad [data directory]. + +This command requires file system permissions to access the data +directory on disk. The Nomad server locks access to the data +directory, so this command cannot be run on a data directory that is +being used by a running Nomad server. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator raft info +``` + +## Examples + +An example output is as follows: + +```shell-session +$ sudo nomad operator raft info /var/nomad/data +path: /var/nomad/data/server/raft/raft.db +length: 10 +first index: 1 +last index: 10 +``` + +[data directory]: /docs/configuration#data_dir diff --git a/website/content/docs/commands/operator/raft-list-peers.mdx b/website/content/docs/commands/operator/raft-list-peers.mdx index f0c44ef63284..63fee11d122e 100644 --- a/website/content/docs/commands/operator/raft-list-peers.mdx +++ b/website/content/docs/commands/operator/raft-list-peers.mdx @@ -7,7 +7,7 @@ description: | # Command: operator raft list-peers -The Raft list-peers command is used to display the current Raft peer +The `raft list-peers` command is used to display the current Raft peer configuration. See the [Outage Recovery] guide for some examples of how this command is used. diff --git a/website/content/docs/commands/operator/raft-logs.mdx b/website/content/docs/commands/operator/raft-logs.mdx new file mode 100644 index 000000000000..ec8c2a53f813 --- /dev/null +++ b/website/content/docs/commands/operator/raft-logs.mdx @@ -0,0 +1,38 @@ +--- +layout: docs +page_title: 'Commands: operator raft logs' +description: | + Display Raft server state. +--- + +# Command: operator raft logs + +The `raft logs` command is used to display the log entries persisted in +the Nomad [data directory] in JSON format. + +This command requires file system permissions to access the data +directory on disk. The Nomad server locks access to the data +directory, so this command cannot be run on a data directory that is +being used by a running Nomad server. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator raft logs [options] +``` + +## Examples + +The output of this command can be very large, so it's recommended that +you redirect the output to a file for later examination with other +tools. + +```shell-session +$ sudo nomad operator raft logs /var/nomad/data > ~/raft-logs.json +$ jq . < ~/raft-logs.json +``` + +[data directory]: /docs/configuration#data_dir diff --git a/website/content/docs/commands/operator/raft-state.mdx b/website/content/docs/commands/operator/raft-state.mdx new file mode 100644 index 000000000000..e97fb8949c71 --- /dev/null +++ b/website/content/docs/commands/operator/raft-state.mdx @@ -0,0 +1,46 @@ +--- +layout: docs +page_title: 'Commands: operator raft state' +description: | + Display Raft server state. +--- + +# Command: operator raft state + +The `raft state` command is used to display the server state obtained by +replaying raft log entries persisted in the Nomad [data directory] in +JSON format. + +This command requires file system permissions to access the data +directory on disk. The Nomad server locks access to the data +directory, so this command cannot be run on a data directory that is +being used by a running Nomad server. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator raft state [options] +``` + +## Raft State Options + +- `-last-index=`: Set the last log index to be applied, to + drop spurious log entries not properly committed. If the + `last_index` option is zero or negative, it's treated as an offset + from the last index seen in raft. + +## Examples + +The output of this command can be very large, so it's recommended that +you redirect the output to a file for later examination with other +tools. + +```shell-session +$ sudo nomad operator raft state /var/nomad/data > ~/raft-state.json +$ jq . < ~/raft-state.json +``` + +[data directory]: /docs/configuration#data_dir diff --git a/website/content/docs/commands/operator/snapshot-state.mdx b/website/content/docs/commands/operator/snapshot-state.mdx new file mode 100644 index 000000000000..e944c049b41f --- /dev/null +++ b/website/content/docs/commands/operator/snapshot-state.mdx @@ -0,0 +1,30 @@ +--- +layout: docs +page_title: 'Commands: operator snapshot state' +description: | + Displays a JSON representation of a Raft snapshot. +--- + +# Command: operator snapshot state + +Displays a JSON representation of state in a raft snapshot on disk. + +~> **Warning:** This is a low-level debugging tool and not subject to + Nomad's usual backward compatibility guarantees. + +## Usage + +```plaintext +nomad operator snapshot state +``` + +## Examples + +The output of this command can be very large, so it's recommended that +you redirect the output to a file for later examination with other +tools. + +```shell-session +$ nomad operator snapshot state backup.snap > ~/raft-state.json +$ jq . < ~/raft-state.json +``` diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 28f67624c5fb..9985a84d995d 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -548,14 +548,26 @@ "title": "metrics", "path": "commands/operator/metrics" }, + { + "title": "raft info", + "path": "commands/operator/raft-info" + }, { "title": "raft list-peers", "path": "commands/operator/raft-list-peers" }, + { + "title": "raft logs", + "path": "commands/operator/raft-logs" + }, { "title": "raft remove-peer", "path": "commands/operator/raft-remove-peer" }, + { + "title": "raft state", + "path": "commands/operator/raft-state" + }, { "title": "snapshot agent", "path": "commands/operator/snapshot-agent" @@ -571,6 +583,10 @@ { "title": "snapshot save", "path": "commands/operator/snapshot-save" + }, + { + "title": "snapshot state", + "path": "commands/operator/snapshot-state" } ] }, From c6dd71322a936d984b173eb0c4094cb710724434 Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Thu, 16 Dec 2021 11:23:05 -0500 Subject: [PATCH 15/40] chore: prettify job-page/summary --- .../components/job-page/parts/summary.hbs | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/ui/app/templates/components/job-page/parts/summary.hbs b/ui/app/templates/components/job-page/parts/summary.hbs index 244e4f7d329e..ebd75dc568d7 100644 --- a/ui/app/templates/components/job-page/parts/summary.hbs +++ b/ui/app/templates/components/job-page/parts/summary.hbs @@ -1,4 +1,10 @@ - +
@@ -14,7 +20,6 @@ {{/if}}
- {{#unless a.isOpen}}
@@ -22,7 +27,9 @@ {{#if (gt a.item.totalChildren 0)}} {{else}} - No Children + + No Children + {{/if}} {{else}} @@ -33,17 +40,25 @@
- {{#component (if a.item.hasChildren "children-status-bar" "allocation-status-bar") - allocationContainer=a.item.summary - job=a.item.summary - class="split-view" as |chart|}} -
    - {{#each chart.data as |datum index|}} -
  1. - -
    2. - {{/each}} -
- {{/component}} + {{#component + (if a.item.hasChildren "children-status-bar" "allocation-status-bar") + allocationContainer=a.item.summary + job=a.item.summary + class="split-view" as |chart| + }} +
    + {{#each chart.data as |datum index|}} +
  1. + +
    2. + {{/each}} +
+ {{/component}} - + \ No newline at end of file From 094c1912f96f3a06e504795f436057f8e998e4ad Mon Sep 17 00:00:00 2001 From: Jai Bhagat Date: Thu, 16 Dec 2021 11:24:03 -0500 Subject: [PATCH 16/40] feat: add sliceClick to job-page/summary --- ui/app/components/job-page/parts/summary.js | 20 ++++++++++++++++++- .../components/job-page/parts/summary.hbs | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ui/app/components/job-page/parts/summary.js b/ui/app/components/job-page/parts/summary.js index 0cb821cc8344..51aace4a4225 100644 --- a/ui/app/components/job-page/parts/summary.js +++ b/ui/app/components/job-page/parts/summary.js @@ -1,14 +1,32 @@ import Component from '@ember/component'; -import { computed } from '@ember/object'; +import { action, computed } from '@ember/object'; +import { inject as service } from '@ember/service'; import { classNames } from '@ember-decorators/component'; import classic from 'ember-classic-decorator'; @classic @classNames('boxed-section') export default class Summary extends Component { + @service router; + job = null; forceCollapsed = false; + @action + gotoAllocations(status) { + this.router.transitionTo('jobs.job.allocations', this.job, { + queryParams: { + status: JSON.stringify(status), + namespace: this.job.get('namespace.name'), + }, + }); + } + + @action + onSliceClick(ev, slice) { + this.gotoAllocations([slice.label.camelize()]); + } + @computed('forceCollapsed') get isExpanded() { if (this.forceCollapsed) return false; diff --git a/ui/app/templates/components/job-page/parts/summary.hbs b/ui/app/templates/components/job-page/parts/summary.hbs index ebd75dc568d7..9b4ed85090fb 100644 --- a/ui/app/templates/components/job-page/parts/summary.hbs +++ b/ui/app/templates/components/job-page/parts/summary.hbs @@ -44,6 +44,7 @@ (if a.item.hasChildren "children-status-bar" "allocation-status-bar") allocationContainer=a.item.summary job=a.item.summary + onSliceClick=this.onSliceClick class="split-view" as |chart| }}
    From bd18a452abf5369309a876f2edf3ab7df983af23 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 16 Dec 2021 13:38:58 -0500 Subject: [PATCH 17/40] cli: stream raft logs to operator raft logs subcommand (#11684) The `nomad operator raft logs` command uses a raft helper that reads in the logs from raft and serializes them to JSON. The previous implementation returned the slice of all logs and then serializes the entire object. Update the helper to stream the log entries and then serialize them as newline-delimited JSON. --- command/operator_raft_logs.go | 53 ++++++++++++++++++++++++++++------- helper/raftutil/state.go | 51 ++++++++++++++++++++------------- 2 files changed, 74 insertions(+), 30 deletions(-) diff --git a/command/operator_raft_logs.go b/command/operator_raft_logs.go index fe3233ee6c9c..006827aa6594 100644 --- a/command/operator_raft_logs.go +++ b/command/operator_raft_logs.go @@ -28,6 +28,11 @@ Usage: nomad operator raft logs This is a low-level debugging tool and not subject to Nomad's usual backward compatibility guarantees. +Raft Logs Options: + + -pretty + By default this command outputs newline delimited JSON. If the -pretty flag + is passed, each entry will be pretty-printed. ` return strings.TrimSpace(helpText) } @@ -47,10 +52,20 @@ func (c *OperatorRaftLogsCommand) Synopsis() string { func (c *OperatorRaftLogsCommand) Name() string { return "operator raft logs" } func (c *OperatorRaftLogsCommand) Run(args []string) int { - if len(args) != 1 { + + var pretty bool + flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) + flagSet.Usage = func() { c.Ui.Output(c.Help()) } + flagSet.BoolVar(&pretty, "pretty", false, "") + + if err := flagSet.Parse(args); err != nil { + return 1 + } + + args = flagSet.Args() + if l := len(args); l != 1 { c.Ui.Error("This command takes one argument: ") c.Ui.Error(commandErrorText(c)) - return 1 } @@ -60,21 +75,39 @@ func (c *OperatorRaftLogsCommand) Run(args []string) int { return 1 } - logs, warnings, err := raftutil.LogEntries(raftPath) + enc := json.NewEncoder(os.Stdout) + if pretty { + enc.SetIndent("", " ") + } + + logChan, warningsChan, err := raftutil.LogEntries(raftPath) if err != nil { c.Ui.Error(err.Error()) return 1 } - for _, warning := range warnings { - c.Ui.Error(warning.Error()) + // so that the warnings don't end up mixed into the JSON stream, + // collect them and print them once we're done + warnings := []error{} + +DONE: + for { + select { + case log := <-logChan: + if log == nil { + break DONE // no more logs, but break to print warnings + } + if err := enc.Encode(log); err != nil { + c.Ui.Error(fmt.Sprintf("failed to encode output: %v", err)) + return 1 + } + case warning := <-warningsChan: + warnings = append(warnings, warning) + } } - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - if err := enc.Encode(logs); err != nil { - c.Ui.Error(fmt.Sprintf("failed to encode output: %v", err)) - return 1 + for _, warning := range warnings { + c.Ui.Error(warning.Error()) } return 0 diff --git a/helper/raftutil/state.go b/helper/raftutil/state.go index 1206fbdba46a..f902a32efbaa 100644 --- a/helper/raftutil/state.go +++ b/helper/raftutil/state.go @@ -32,33 +32,44 @@ func RaftStateInfo(p string) (store *raftboltdb.BoltStore, firstIdx uint64, last return s, firstIdx, lastIdx, nil } -// LogEntries returns the log entries as found in raft log in the passed data-dir directory -func LogEntries(p string) (logs []interface{}, warnings []error, err error) { +// LogEntries reads the raft logs found in the data directory found at +// the path `p`, and returns a channel of logs, and a channel of +// warnings. If opening the raft state returns an error, both channels +// will be nil. +func LogEntries(p string) (<-chan interface{}, <-chan error, error) { store, firstIdx, lastIdx, err := RaftStateInfo(p) if err != nil { return nil, nil, fmt.Errorf("failed to open raft logs: %v", err) } - defer store.Close() - result := make([]interface{}, 0, lastIdx-firstIdx+1) - for i := firstIdx; i <= lastIdx; i++ { - var e raft.Log - err := store.GetLog(i, &e) - if err != nil { - warnings = append(warnings, fmt.Errorf("failed to read log entry at index %d (firstIdx: %d, lastIdx: %d): %v", i, firstIdx, lastIdx, err)) - continue - } - - m, err := decode(&e) - if err != nil { - warnings = append(warnings, fmt.Errorf("failed to decode log entry at index %d: %v", i, err)) - continue + entries := make(chan interface{}) + warnings := make(chan error) + + go func() { + defer store.Close() + defer close(entries) + for i := firstIdx; i <= lastIdx; i++ { + var e raft.Log + err := store.GetLog(i, &e) + if err != nil { + warnings <- fmt.Errorf( + "failed to read log entry at index %d (firstIdx: %d, lastIdx: %d): %v", + i, firstIdx, lastIdx, err) + continue + } + + entry, err := decode(&e) + if err != nil { + warnings <- fmt.Errorf( + "failed to decode log entry at index %d: %v", i, err) + continue + } + + entries <- entry } + }() - result = append(result, m) - } - - return result, warnings, nil + return entries, warnings, nil } type logMessage struct { From 495a46ee79c24a7c2e64ee7070f4622fd728bbac Mon Sep 17 00:00:00 2001 From: Noel Quiles <3746694+EnMod@users.noreply.github.com> Date: Thu, 16 Dec 2021 13:43:47 -0500 Subject: [PATCH 18/40] website: Disable alert banner (#11688) --- website/data/alert-banner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/data/alert-banner.js b/website/data/alert-banner.js index 162cb6895b82..ce39c7cbee04 100644 --- a/website/data/alert-banner.js +++ b/website/data/alert-banner.js @@ -1,4 +1,4 @@ -export const ALERT_BANNER_ACTIVE = true +export const ALERT_BANNER_ACTIVE = false // https://github.com/hashicorp/web-components/tree/master/packages/alert-banner export default { tag: 'Blog post', From 15db86a6af4c38976c769823c7752a0189ddd269 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Thu, 16 Dec 2021 14:14:01 -0500 Subject: [PATCH 19/40] docs: add more references and examples to the `template` block (#11691) --- .../docs/job-specification/template.mdx | 83 +++++++++++++++++-- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/website/content/docs/job-specification/template.mdx b/website/content/docs/job-specification/template.mdx index c5f2f938d675..d47e6bbf52bb 100644 --- a/website/content/docs/job-specification/template.mdx +++ b/website/content/docs/job-specification/template.mdx @@ -32,11 +32,16 @@ job "docs" { } ``` -Nomad utilizes a tool called [Consul Template][ct]. Since Nomad v0.5.3, the -template can reference [Nomad's runtime environment variables][env]. Since Nomad -v0.5.6, the template can reference [Node attributes and metadata][nodevars]. For -a full list of the API template functions, please refer to the [Consul Template -README][ct]. Since Nomad v0.6.0, templates can be read as environment variables. +Nomad utilizes [Go template][gt] and a tool called [Consul Template][ct], which +adds a set of new functions that can be used to retrieve data from Consul and +Vault. Since Nomad v0.5.3, the template can reference [Nomad's runtime +environment variables][env], and since Nomad v0.5.6, the template can reference +[Node attributes and metadata][nodevars]. Since Nomad v0.6.0, templates can be +read as environment variables. + +For a full list of the API template functions, please refer to the [Consul +Template documentation][ct_api]. For a an introduction to Go templates, please +refer to the [Learn Go Template Syntax][gt_learn] Learn guide. ## `template` Parameters @@ -257,6 +262,65 @@ task "task" { } ``` +## Consul Integration + +### Consul KV + +Consul KV values can be accessed using the [`key`][ct_api_key] function to +retrieve a single value from a key path. The [`ls`][ct_api_ls] function can be +used to retrieve all keys in a path. For deeply nested paths, use the +[`tree`][ct_api_tree] function. + +```hcl + template { + data = < Date: Thu, 16 Dec 2021 14:32:20 -0500 Subject: [PATCH 20/40] docs: add v1.2.0 upgrade guide about Nomad UI ACL change for job details page (#11689) --- website/content/docs/upgrade/upgrade-specific.mdx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 90b5a95a2b92..55dc289716d7 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -38,6 +38,16 @@ this problem. The Nvidia device is now an external plugin and must be installed separately. Refer to [the Nvidia device plugin's documentation][nvidia] for details. +#### ACL requirements for accessing the job details page in the Nomad UI + +Nomad 1.2.0 introduced a new UI component to display the status of `system` and +`sysbatch` jobs in each client where they are running. This feature makes an +API call to an endpoint that requires `node:read` ACL permission. Tokens used +to access the Nomad UI will need to be updated to include this permission in +order to access a job details page. + +This was an unintended change and will be fixed in a future release. + ## Nomad 1.0.11 and 1.1.5 Enterprise #### Audit log file names From fa3de735cfd9a1ca3393daa96889036925d1f6b0 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 16 Dec 2021 11:41:01 -0800 Subject: [PATCH 21/40] cli: return error from raft commands if db is open Before this change trying to run `nomad operator raft {info,logs}` on an inuse raft.db would cause the command to block until the agent using raft.db is closed. After this change the command will block for 1s before returning a (hopefully) helpful error message. This change also sets the ReadOnly mode on the underlying BoltDb to ensure diagnostics make no changes to the underlying store. We have no evidence this has ever occurred, but it seems like a useful safety measure. No changelog added since this is a minor tweak in a "new" feature (it was hidden in previous relases). --- helper/raftutil/state.go | 20 +++++++++++++- helper/raftutil/state_test.go | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 helper/raftutil/state_test.go diff --git a/helper/raftutil/state.go b/helper/raftutil/state.go index f902a32efbaa..68c26b5c0fcc 100644 --- a/helper/raftutil/state.go +++ b/helper/raftutil/state.go @@ -2,20 +2,38 @@ package raftutil import ( "bytes" + "errors" "fmt" "os" "path/filepath" + "strings" + "time" + "github.com/boltdb/bolt" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/raft" raftboltdb "github.com/hashicorp/raft-boltdb" ) +var ( + errAlreadyOpen = errors.New("unable to open raft logs that are in use") +) + // RaftStateInfo returns info about the nomad state, as found in the passed data-dir directory func RaftStateInfo(p string) (store *raftboltdb.BoltStore, firstIdx uint64, lastIdx uint64, err error) { - s, err := raftboltdb.NewBoltStore(p) + opts := raftboltdb.Options{ + Path: p, + BoltOptions: &bolt.Options{ + ReadOnly: true, + Timeout: 1 * time.Second, + }, + } + s, err := raftboltdb.New(opts) if err != nil { + if strings.HasSuffix(err.Error(), "timeout") { + return nil, 0, 0, errAlreadyOpen + } return nil, 0, 0, fmt.Errorf("failed to open raft logs: %v", err) } diff --git a/helper/raftutil/state_test.go b/helper/raftutil/state_test.go new file mode 100644 index 000000000000..ea8075ca31a4 --- /dev/null +++ b/helper/raftutil/state_test.go @@ -0,0 +1,52 @@ +package raftutil + +import ( + "path/filepath" + "testing" + + raftboltdb "github.com/hashicorp/raft-boltdb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRaftStateInfo_InUse asserts that commands that inspect raft +// state such as "nomad operator raft info" and "nomad operator raft +// logs" fail with a helpful error message when called on an inuse +// database. +func TestRaftStateInfo_InUse(t *testing.T) { + t.Parallel() // since there's a 1s timeout. + + // First create an empty raft db + dir := filepath.Join(t.TempDir(), "raft.db") + + fakedb, err := raftboltdb.NewBoltStore(dir) + require.NoError(t, err) + + // Next try to read the db without closing it + s, _, _, err := RaftStateInfo(dir) + assert.Nil(t, s) + require.EqualError(t, err, errAlreadyOpen.Error()) + + // LogEntries should produce the same error + _, _, err = LogEntries(dir) + require.EqualError(t, err, "failed to open raft logs: "+errAlreadyOpen.Error()) + + // Commands should work once the db is closed + require.NoError(t, fakedb.Close()) + + s, _, _, err = RaftStateInfo(dir) + assert.NotNil(t, s) + require.NoError(t, err) + require.NoError(t, s.Close()) + + logCh, errCh, err := LogEntries(dir) + require.NoError(t, err) + + // Consume entries to cleanly close db + for closed := false; closed; { + select { + case _, closed = <-logCh: + case <-errCh: + } + } +} From a8854bc3a827cee0522755c3f64a441c35527f8e Mon Sep 17 00:00:00 2001 From: Jai <41024828+ChaiWithJai@users.noreply.github.com> Date: Fri, 17 Dec 2021 09:36:42 -0500 Subject: [PATCH 22/40] fix: remove eslint disable indent --- ui/app/controllers/jobs/job/allocations.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index 7c5c289378e8..2b1861ea8314 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -12,7 +12,6 @@ import classic from 'ember-classic-decorator'; @classic export default class AllocationsController extends Controller.extend( - /*eslint-disable indent */ Sortable, Searchable, WithNamespaceResetting From c0add56610b23ee2cdfb53f2a31100eb67b962e4 Mon Sep 17 00:00:00 2001 From: Jai <41024828+ChaiWithJai@users.noreply.github.com> Date: Fri, 17 Dec 2021 09:46:29 -0500 Subject: [PATCH 23/40] fix: more descriptive parameters in sort function Co-authored-by: Luiz Aoqui --- ui/app/controllers/clients/client/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/app/controllers/clients/client/index.js b/ui/app/controllers/clients/client/index.js index a1d6f37b0ebc..d6da57e688ac 100644 --- a/ui/app/controllers/clients/client/index.js +++ b/ui/app/controllers/clients/client/index.js @@ -201,7 +201,7 @@ export default class ClientController extends Controller.extend(Sortable, Search this.set('qpTaskGroup', serialize(intersection(taskGroups, this.selectionTaskGroup))); }); - return taskGroups.sort().map(dc => ({ key: dc, label: dc })); + return taskGroups.sort().map(tg => ({ key: tg, label: tg })); } @action From 6112620590d4af6d4f0ca55830a592c26949217f Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 16:55:40 -0500 Subject: [PATCH 24/40] ui: fix client details page alloc status filter and replace task group with namespace and job --- ui/app/controllers/clients/client/index.js | 63 +++++--- ui/app/models/allocation.js | 6 + ui/app/templates/clients/client/index.hbs | 21 ++- ui/tests/acceptance/client-detail-test.js | 165 ++++++++++++++++++++- ui/tests/pages/clients/detail.js | 7 + 5 files changed, 237 insertions(+), 25 deletions(-) diff --git a/ui/app/controllers/clients/client/index.js b/ui/app/controllers/clients/client/index.js index d6da57e688ac..b3cc6fea0000 100644 --- a/ui/app/controllers/clients/client/index.js +++ b/ui/app/controllers/clients/client/index.js @@ -32,18 +32,22 @@ export default class ClientController extends Controller.extend(Sortable, Search onlyPreemptions: 'preemptions', }, { - qpStatus: 'status', + qpNamespace: 'namespace', + }, + { + qpJob: 'job', }, { - qpTaskGroup: 'taskGroup', + qpStatus: 'status', }, ]; // Set in the route flagAsDraining = false; + qpNamespace = ''; + qpJob = ''; qpStatus = ''; - qpTaskGroup = ''; currentPage = 1; pageSize = 8; @@ -61,18 +65,22 @@ export default class ClientController extends Controller.extend(Sortable, Search 'model.allocations.[]', 'preemptions.[]', 'onlyPreemptions', - 'selectionStatus', - 'selectionTaskGroup' + 'selectionNamespace', + 'selectionJob', + 'selectionStatus' ) get visibleAllocations() { const allocations = this.onlyPreemptions ? this.preemptions : this.model.allocations; - const { selectionStatus, selectionTaskGroup } = this; + const { selectionNamespace, selectionJob, selectionStatus } = this; return allocations.filter(alloc => { - if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { + if (selectionNamespace.length && !selectionNamespace.includes(alloc.get('namespace'))) { + return false; + } + if (selectionJob.length && !selectionJob.includes(alloc.get('plainJobId'))) { return false; } - if (selectionTaskGroup.length && !selectionTaskGroup.includes(alloc.taskGroupName)) { + if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { return false; } return true; @@ -83,8 +91,9 @@ export default class ClientController extends Controller.extend(Sortable, Search @alias('listSorted') listToSearch; @alias('listSearched') sortedAllocations; + @selection('qpNamespace') selectionNamespace; + @selection('qpJob') selectionJob; @selection('qpStatus') selectionStatus; - @selection('qpTaskGroup') selectionTaskGroup; eligibilityError = null; stopDrainError = null; @@ -182,8 +191,7 @@ export default class ClientController extends Controller.extend(Sortable, Search get optionsAllocationStatus() { return [ - { key: 'queued', label: 'Queued' }, - { key: 'starting', label: 'Starting' }, + { key: 'pending', label: 'Pending' }, { key: 'running', label: 'Running' }, { key: 'complete', label: 'Complete' }, { key: 'failed', label: 'Failed' }, @@ -191,17 +199,38 @@ export default class ClientController extends Controller.extend(Sortable, Search ]; } - @computed('model.allocations.[]', 'selectionTaskGroup') - get optionsTaskGroups() { - const taskGroups = Array.from(new Set(this.model.allocations.mapBy('taskGroupName'))).compact(); + @computed('model.allocations.[]', 'selectionJob', 'selectionNamespace') + get optionsJob() { + // Only show options for jobs in the selected namespaces, if any. + const ns = this.selectionNamespace; + const jobs = Array.from( + new Set( + this.model.allocations + .filter(a => ns.length === 0 || ns.includes(a.namespace)) + .mapBy('plainJobId') + ) + ).compact(); + + // Update query param when the list of jobs changes. + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpJob', serialize(intersection(jobs, this.selectionJob))); + }); + + return jobs.sort().map(job => ({ key: job, label: job })); + } + + @computed('model.allocations.[]', 'selectionNamespace') + get optionsNamespace() { + const ns = Array.from(new Set(this.model.allocations.mapBy('namespace'))).compact(); - // Update query param when the list of clients changes. + // Update query param when the list of namespaces changes. scheduleOnce('actions', () => { // eslint-disable-next-line ember/no-side-effects - this.set('qpTaskGroup', serialize(intersection(taskGroups, this.selectionTaskGroup))); + this.set('qpNamespace', serialize(intersection(ns, this.selectionNamespace))); }); - return taskGroups.sort().map(tg => ({ key: tg, label: tg })); + return ns.sort().map(n => ({ key: n, label: n })); } @action diff --git a/ui/app/models/allocation.js b/ui/app/models/allocation.js index f2293d3fc868..028ee23bf406 100644 --- a/ui/app/models/allocation.js +++ b/ui/app/models/allocation.js @@ -23,6 +23,7 @@ export default class Allocation extends Model { @shortUUIDProperty('id') shortId; @belongsTo('job') job; @belongsTo('node') node; + @attr('string') namespace; @attr('string') name; @attr('string') taskGroupName; @fragment('resources') resources; @@ -38,6 +39,11 @@ export default class Allocation extends Model { @attr('string') clientStatus; @attr('string') desiredStatus; + @computed('') + get plainJobId() { + return JSON.parse(this.belongsTo('job').id())[0]; + } + @computed('clientStatus') get statusIndex() { return STATUS_ORDER[this.clientStatus] || 100; diff --git a/ui/app/templates/clients/client/index.hbs b/ui/app/templates/clients/client/index.hbs index 3ccb8de5d6f2..429cbce55fc6 100644 --- a/ui/app/templates/clients/client/index.hbs +++ b/ui/app/templates/clients/client/index.hbs @@ -295,6 +295,20 @@ {{/if}}
+ + - selection.includes(alloc.jobId), + }); + + testFacet('Status', { + facet: ClientDetail.facets.status, + paramName: 'status', + expectedOptions: ['Pending', 'Running', 'Complete', 'Failed', 'Lost'], + async beforeEach() { + server.createList('job', 5, { createAllocations: false }); + ['pending', 'running', 'complete', 'failed', 'lost'].forEach(s => { + server.createList('allocation', 5, { clientStatus: s }); + }); + + await ClientDetail.visit({ id: node.id }); + }, + filter: (alloc, selection) => selection.includes(alloc.clientStatus), + }); }); module('Acceptance | client detail (multi-namespace)', function(hooks) { @@ -1018,7 +1046,11 @@ module('Acceptance | client detail (multi-namespace)', function(hooks) { // Make a job for each namespace, but have both scheduled on the same node server.create('job', { id: 'job-1', namespaceId: 'default', createAllocations: false }); - server.createList('allocation', 3, { nodeId: node.id, clientStatus: 'running' }); + server.createList('allocation', 3, { + nodeId: node.id, + jobId: 'job-1', + clientStatus: 'running', + }); server.create('job', { id: 'job-2', namespaceId: 'other-namespace', createAllocations: false }); server.createList('allocation', 3, { @@ -1047,4 +1079,135 @@ module('Acceptance | client detail (multi-namespace)', function(hooks) { 'Job Two fetched correctly' ); }); + + testFacet('Namespace', { + facet: ClientDetail.facets.namespace, + paramName: 'namespace', + expectedOptions(allocs) { + return Array.from(new Set(allocs.mapBy('namespace'))).sort(); + }, + async beforeEach() { + await ClientDetail.visit({ id: node.id }); + }, + filter: (alloc, selection) => selection.includes(alloc.namespace), + }); + + test('facet Namespace | selecting namespace filters job options', async function(assert) { + await ClientDetail.visit({ id: node.id }); + + const nsFacet = ClientDetail.facets.namespace; + const jobFacet = ClientDetail.facets.job; + + // Select both namespaces. + await nsFacet.toggle(); + await nsFacet.options.objectAt(0).toggle(); + await nsFacet.options.objectAt(1).toggle(); + await jobFacet.toggle(); + + assert.deepEqual( + jobFacet.options.map(option => option.label.trim()), + ['job-1', 'job-2'] + ); + + // Select juse one namespace. + await nsFacet.toggle(); + await nsFacet.options.objectAt(1).toggle(); // deselect second option + await jobFacet.toggle(); + + assert.deepEqual( + jobFacet.options.map(option => option.label.trim()), + ['job-1'] + ); + }); }); + +function testFacet(label, { facet, paramName, beforeEach, filter, expectedOptions }) { + test(`facet ${label} | the ${label} facet has the correct options`, async function(assert) { + await beforeEach(); + await facet.toggle(); + + let expectation; + if (typeof expectedOptions === 'function') { + expectation = expectedOptions(server.db.allocations); + } else { + expectation = expectedOptions; + } + + assert.deepEqual( + facet.options.map(option => option.label.trim()), + expectation, + 'Options for facet are as expected' + ); + }); + + test(`facet ${label} | the ${label} facet filters the allocations list by ${label}`, async function(assert) { + let option; + + await beforeEach(); + + await facet.toggle(); + option = facet.options.objectAt(0); + await option.toggle(); + + const selection = [option.key]; + const expectedAllocs = server.db.allocations + .filter(alloc => filter(alloc, selection)) + .sortBy('modifyIndex') + .reverse(); + + ClientDetail.allocations.forEach((alloc, index) => { + assert.equal( + alloc.id, + expectedAllocs[index].id, + `Allocation at ${index} is ${expectedAllocs[index].id}` + ); + }); + }); + + test(`facet ${label} | selecting multiple options in the ${label} facet results in a broader search`, async function(assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.key); + await option2.toggle(); + selection.push(option2.key); + + const expectedAllocs = server.db.allocations + .filter(alloc => filter(alloc, selection)) + .sortBy('modifyIndex') + .reverse(); + + ClientDetail.allocations.forEach((alloc, index) => { + assert.equal( + alloc.id, + expectedAllocs[index].id, + `Allocation at ${index} is ${expectedAllocs[index].id}` + ); + }); + }); + + test(`facet ${label} | selecting options in the ${label} facet updates the ${paramName} query param`, async function(assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.key); + await option2.toggle(); + selection.push(option2.key); + + assert.equal( + currentURL(), + `/clients/${node.id}?${paramName}=${encodeURIComponent(JSON.stringify(selection))}`, + 'URL has the correct query param key and value' + ); + }); +} diff --git a/ui/tests/pages/clients/detail.js b/ui/tests/pages/clients/detail.js index 70afd69a7d9d..daa6e647aa0a 100644 --- a/ui/tests/pages/clients/detail.js +++ b/ui/tests/pages/clients/detail.js @@ -14,6 +14,7 @@ import allocations from 'nomad-ui/tests/pages/components/allocations'; import twoStepButton from 'nomad-ui/tests/pages/components/two-step-button'; import notification from 'nomad-ui/tests/pages/components/notification'; import toggle from 'nomad-ui/tests/pages/components/toggle'; +import { multiFacet } from 'nomad-ui/tests/pages/components/facet'; export default create({ visit: visitable('/clients/:id'), @@ -45,6 +46,12 @@ export default create({ allCount: text('[data-test-filter-all]'), }, + facets: { + namespace: multiFacet('[data-test-allocation-namespace-facet]'), + job: multiFacet('[data-test-allocation-job-facet]'), + status: multiFacet('[data-test-allocation-status-facet]'), + }, + attributesTable: isPresent('[data-test-attributes]'), metaTable: isPresent('[data-test-meta]'), emptyMetaMessage: isPresent('[data-test-empty-meta-message]'), From 648b71c96aa727a9dc5b1bf09710ea75ce48716b Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 17:38:04 -0500 Subject: [PATCH 25/40] ui: display empty message in the client details page if there are no allocations to show --- ui/app/controllers/clients/client/index.js | 19 ++--- ui/app/templates/clients/client/index.hbs | 99 +++++++++++++--------- ui/tests/acceptance/client-detail-test.js | 22 +++++ ui/tests/pages/clients/detail.js | 6 ++ 4 files changed, 95 insertions(+), 51 deletions(-) diff --git a/ui/app/controllers/clients/client/index.js b/ui/app/controllers/clients/client/index.js index b3cc6fea0000..a918a7b8298b 100644 --- a/ui/app/controllers/clients/client/index.js +++ b/ui/app/controllers/clients/client/index.js @@ -61,19 +61,16 @@ export default class ClientController extends Controller.extend(Sortable, Search onlyPreemptions = false; - @computed( - 'model.allocations.[]', - 'preemptions.[]', - 'onlyPreemptions', - 'selectionNamespace', - 'selectionJob', - 'selectionStatus' - ) + @computed('model.allocations.[]', 'preemptions.[]', 'onlyPreemptions') get visibleAllocations() { - const allocations = this.onlyPreemptions ? this.preemptions : this.model.allocations; + return this.onlyPreemptions ? this.preemptions : this.model.allocations; + } + + @computed('visibleAllocations.[]', 'selectionNamespace', 'selectionJob', 'selectionStatus') + get filteredAllocations() { const { selectionNamespace, selectionJob, selectionStatus } = this; - return allocations.filter(alloc => { + return this.visibleAllocations.filter(alloc => { if (selectionNamespace.length && !selectionNamespace.includes(alloc.get('namespace'))) { return false; } @@ -87,7 +84,7 @@ export default class ClientController extends Controller.extend(Sortable, Search }); } - @alias('visibleAllocations') listToSort; + @alias('filteredAllocations') listToSort; @alias('listSorted') listToSearch; @alias('listSearched') sortedAllocations; diff --git a/ui/app/templates/clients/client/index.hbs b/ui/app/templates/clients/client/index.hbs index 429cbce55fc6..7e623678c400 100644 --- a/ui/app/templates/clients/client/index.hbs +++ b/ui/app/templates/clients/client/index.hbs @@ -325,47 +325,66 @@ />
-
- - - - - ID - Created - Modified - Status - Job - Version - Volume - CPU - Memory - - - - - -
- +
+ {{#if this.sortedAllocations.length}} + + + + + ID + Created + Modified + Status + Job + Version + Volume + CPU + Memory + + + + + +
+ +
+ + {{else}} +
+ {{#if (eq this.visibleAllocations.length 0)}} +

No Allocations

+

+ The node doesn't have any allocations. +

+ {{else if this.searchTerm}} +

No Matches

+

No allocations match the term {{this.searchTerm}}

+ {{else if (eq this.sortedAllocations.length 0)}} +

No Matches

+

+ No allocations match your current filter selection. +

+ {{/if}}
- + {{/if}}
diff --git a/ui/tests/acceptance/client-detail-test.js b/ui/tests/acceptance/client-detail-test.js index f44718255291..cfd9042adcb2 100644 --- a/ui/tests/acceptance/client-detail-test.js +++ b/ui/tests/acceptance/client-detail-test.js @@ -130,6 +130,15 @@ module('Acceptance | client detail', function(hooks) { ); }); + test('/clients/:id should show empty message if there are no allocations on the node', async function(assert) { + const emptyNode = server.create('node'); + + await ClientDetail.visit({ id: emptyNode.id }); + + assert.true(ClientDetail.emptyAllocations.isVisible, 'Empty message is visible'); + assert.equal(ClientDetail.emptyAllocations.headline, 'No Allocations'); + }); + test('each allocation should have high-level details for the allocation', async function(assert) { const allocation = server.db.allocations .where({ nodeId: node.id }) @@ -1028,6 +1037,19 @@ module('Acceptance | client detail', function(hooks) { }, filter: (alloc, selection) => selection.includes(alloc.clientStatus), }); + + test('fiter results with no matches display empty message', async function(assert) { + const job = server.create('job', { createAllocations: false }); + server.create('allocation', { jobId: job.id, clientStatus: 'running' }); + + await ClientDetail.visit({ id: node.id }); + const statusFacet = ClientDetail.facets.status; + await statusFacet.toggle(); + await statusFacet.options.objectAt(0).toggle(); + + assert.true(ClientDetail.emptyAllocations.isVisible); + assert.equal(ClientDetail.emptyAllocations.headline, 'No Matches'); + }); }); module('Acceptance | client detail (multi-namespace)', function(hooks) { diff --git a/ui/tests/pages/clients/detail.js b/ui/tests/pages/clients/detail.js index daa6e647aa0a..bcfe0d2de28c 100644 --- a/ui/tests/pages/clients/detail.js +++ b/ui/tests/pages/clients/detail.js @@ -39,6 +39,12 @@ export default create({ ...allocations(), + emptyAllocations: { + scope: '[data-test-empty-allocations-list]', + headline: text('[data-test-empty-allocations-list-headline]'), + body: text('[data-test-empty-allocations-list-body]'), + }, + allocationFilter: { preemptions: clickable('[data-test-filter-preemptions]'), all: clickable('[data-test-filter-all]'), From 1d773d0d9e3e624deb7707b44d59515f9af29c4a Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 18:45:31 -0500 Subject: [PATCH 26/40] ui: fix task group alloc filter and add tests --- ui/app/controllers/jobs/job/task-group.js | 25 ++-- ui/app/templates/jobs/job/task-group.hbs | 2 +- ui/tests/acceptance/task-group-detail-test.js | 140 ++++++++++++++++++ ui/tests/pages/jobs/job/task-group.js | 6 + 4 files changed, 160 insertions(+), 13 deletions(-) diff --git a/ui/app/controllers/jobs/job/task-group.js b/ui/app/controllers/jobs/job/task-group.js index b0fdb6d58cc1..807fa71725ed 100644 --- a/ui/app/controllers/jobs/job/task-group.js +++ b/ui/app/controllers/jobs/job/task-group.js @@ -13,10 +13,10 @@ import classic from 'ember-classic-decorator'; @classic export default class TaskGroupController extends Controller.extend( - Sortable, - Searchable, - WithNamespaceResetting - ) { + Sortable, + Searchable, + WithNamespaceResetting +) { @service userSettings; @service can; @@ -54,14 +54,16 @@ export default class TaskGroupController extends Controller.extend( return ['shortId', 'name']; } - @computed('model.allocations.[]', 'selectionStatus', 'selectionClient') + @computed('model.allocations.[]') get allocations() { - const allocations = this.get('model.allocations') || []; - const { selectionStatus, selectionClient } = this; + return this.get('model.allocations') || []; + } - if (!allocations.length) return allocations; + @computed('allocations.[]', 'selectionStatus', 'selectionClient') + get filteredAllocations() { + const { selectionStatus, selectionClient } = this; - return allocations.filter(alloc => { + return this.allocations.filter(alloc => { if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { return false; } @@ -73,7 +75,7 @@ export default class TaskGroupController extends Controller.extend( }); } - @alias('allocations') listToSort; + @alias('filteredAllocations') listToSort; @alias('listSorted') listToSearch; @alias('listSearched') sortedAllocations; @@ -115,8 +117,7 @@ export default class TaskGroupController extends Controller.extend( get optionsAllocationStatus() { return [ - { key: 'queued', label: 'Queued' }, - { key: 'starting', label: 'Starting' }, + { key: 'pending', label: 'Pending' }, { key: 'running', label: 'Running' }, { key: 'complete', label: 'Complete' }, { key: 'failed', label: 'Failed' }, diff --git a/ui/app/templates/jobs/job/task-group.hbs b/ui/app/templates/jobs/job/task-group.hbs index 87ef5d87a1ff..2c00dcfc2d63 100644 --- a/ui/app/templates/jobs/job/task-group.hbs +++ b/ui/app/templates/jobs/job/task-group.hbs @@ -78,7 +78,7 @@ @onSelect={{action "setFacetQueryParam" "qpStatus"}} /> ev.count == null).length ); }); + + testFacet('Status', { + facet: TaskGroup.facets.status, + paramName: 'status', + expectedOptions: ['Pending', 'Running', 'Complete', 'Failed', 'Lost'], + async beforeEach() { + ['pending', 'running', 'complete', 'failed', 'lost'].forEach(s => { + server.createList('allocation', 5, { clientStatus: s }); + }); + await TaskGroup.visit({ id: job.id, name: taskGroup.name }); + }, + filter: (alloc, selection) => + alloc.jobId == job.id && + alloc.taskGroup == taskGroup.name && + selection.includes(alloc.clientStatus), + }); + + testFacet('Client', { + facet: TaskGroup.facets.client, + paramName: 'client', + expectedOptions(allocs) { + return Array.from( + new Set( + allocs + .filter(alloc => alloc.jobId == job.id && alloc.taskGroup == taskGroup.name) + .mapBy('nodeId') + .map(id => id.split('-')[0]) + ) + ).sort(); + }, + async beforeEach() { + const nodes = server.createList('node', 3, 'forceIPv4'); + nodes.forEach(node => + server.createList('allocation', 5, { + nodeId: node.id, + jobId: job.id, + taskGroup: taskGroup.name, + }) + ); + await TaskGroup.visit({ id: job.id, name: taskGroup.name }); + }, + filter: (alloc, selection) => + alloc.jobId == job.id && + alloc.taskGroup == taskGroup.name && + selection.includes(alloc.nodeId.split('-')[0]), + }); }); + +function testFacet(label, { facet, paramName, beforeEach, filter, expectedOptions }) { + test(`facet ${label} | the ${label} facet has the correct options`, async function(assert) { + await beforeEach(); + await facet.toggle(); + + let expectation; + if (typeof expectedOptions === 'function') { + expectation = expectedOptions(server.db.allocations); + } else { + expectation = expectedOptions; + } + + assert.deepEqual( + facet.options.map(option => option.label.trim()), + expectation, + 'Options for facet are as expected' + ); + }); + + test(`facet ${label} | the ${label} facet filters the allocations list by ${label}`, async function(assert) { + let option; + + await beforeEach(); + + await facet.toggle(); + option = facet.options.objectAt(0); + await option.toggle(); + + const selection = [option.key]; + const expectedAllocs = server.db.allocations + .filter(alloc => filter(alloc, selection)) + .sortBy('modifyIndex') + .reverse(); + + TaskGroup.allocations.forEach((alloc, index) => { + assert.equal( + alloc.id, + expectedAllocs[index].id, + `Allocation at ${index} is ${expectedAllocs[index].id}` + ); + }); + }); + + test(`facet ${label} | selecting multiple options in the ${label} facet results in a broader search`, async function(assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.key); + await option2.toggle(); + selection.push(option2.key); + + const expectedAllocs = server.db.allocations + .filter(alloc => filter(alloc, selection)) + .sortBy('modifyIndex') + .reverse(); + + TaskGroup.allocations.forEach((alloc, index) => { + assert.equal( + alloc.id, + expectedAllocs[index].id, + `Allocation at ${index} is ${expectedAllocs[index].id}` + ); + }); + }); + + test(`facet ${label} | selecting options in the ${label} facet updates the ${paramName} query param`, async function(assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.key); + await option2.toggle(); + selection.push(option2.key); + + assert.equal( + currentURL(), + `/jobs/${job.id}/${taskGroup.name}?${paramName}=${encodeURIComponent( + JSON.stringify(selection) + )}`, + 'URL has the correct query param key and value' + ); + }); +} diff --git a/ui/tests/pages/jobs/job/task-group.js b/ui/tests/pages/jobs/job/task-group.js index 192ea152a2fb..28bd08dbd216 100644 --- a/ui/tests/pages/jobs/job/task-group.js +++ b/ui/tests/pages/jobs/job/task-group.js @@ -13,6 +13,7 @@ import error from 'nomad-ui/tests/pages/components/error'; import pageSizeSelect from 'nomad-ui/tests/pages/components/page-size-select'; import stepperInput from 'nomad-ui/tests/pages/components/stepper-input'; import LifecycleChart from 'nomad-ui/tests/pages/components/lifecycle-chart'; +import { multiFacet } from 'nomad-ui/tests/pages/components/facet'; export default create({ pageSize: 25, @@ -33,6 +34,11 @@ export default create({ isEmpty: isPresent('[data-test-empty-allocations-list]'), + facets: { + status: multiFacet('[data-test-allocation-status-facet]'), + client: multiFacet('[data-test-allocation-client-facet]'), + }, + lifecycleChart: LifecycleChart, hasVolumes: isPresent('[data-test-volumes]'), From ba1151198e2e2db3cb61ddf606be8a35cc4ee7c3 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 18:47:25 -0500 Subject: [PATCH 27/40] changelog: add entry for #11545 --- .changelog/11545.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/11545.txt diff --git a/.changelog/11545.txt b/.changelog/11545.txt new file mode 100644 index 000000000000..427dcbcea838 --- /dev/null +++ b/.changelog/11545.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: Add filters to the allocation list in the client and task group details pages +``` From 770bb0534ad80cb8abb28f872b0d7a64e3e0864f Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 18:55:41 -0500 Subject: [PATCH 28/40] ui: fix linting --- ui/app/controllers/jobs/job/task-group.js | 8 ++++---- ui/tests/acceptance/task-group-detail-test.js | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ui/app/controllers/jobs/job/task-group.js b/ui/app/controllers/jobs/job/task-group.js index 807fa71725ed..796aab4ce3af 100644 --- a/ui/app/controllers/jobs/job/task-group.js +++ b/ui/app/controllers/jobs/job/task-group.js @@ -13,10 +13,10 @@ import classic from 'ember-classic-decorator'; @classic export default class TaskGroupController extends Controller.extend( - Sortable, - Searchable, - WithNamespaceResetting -) { + Sortable, + Searchable, + WithNamespaceResetting + ) { @service userSettings; @service can; diff --git a/ui/tests/acceptance/task-group-detail-test.js b/ui/tests/acceptance/task-group-detail-test.js index 1c9a4d63efec..795a227ee231 100644 --- a/ui/tests/acceptance/task-group-detail-test.js +++ b/ui/tests/acceptance/task-group-detail-test.js @@ -13,7 +13,6 @@ import TaskGroup from 'nomad-ui/tests/pages/jobs/job/task-group'; import Layout from 'nomad-ui/tests/pages/layout'; import pageSizeSelect from './behaviors/page-size-select'; import moment from 'moment'; -import { pauseTest } from '@ember/test-helpers/setup-context'; let job; let taskGroup; From 3f363938b7fe2184fb9bd6376edbf92547c3b43e Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 18:57:54 -0500 Subject: [PATCH 29/40] changelog: fix entry for #11544 --- .changelog/11544.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/11544.txt b/.changelog/11544.txt index 6c48985ccd8b..3daa7ccd784c 100644 --- a/.changelog/11544.txt +++ b/.changelog/11544.txt @@ -1,3 +1,3 @@ ```release-note:feature -ui: feat: add filters to allocations table in jobs/job/allocation view +ui: Add filters to allocations table in jobs/job/allocation view ``` From f8709ff55a9414faf780cb24381293b82b36a5ee Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 19:47:25 -0500 Subject: [PATCH 30/40] ui: fix file formating --- ui/app/templates/jobs/job/allocations.hbs | 83 ++++++----------------- 1 file changed, 21 insertions(+), 62 deletions(-) diff --git a/ui/app/templates/jobs/job/allocations.hbs b/ui/app/templates/jobs/job/allocations.hbs index bcb76b60faa9..22f3aaac8a0e 100644 --- a/ui/app/templates/jobs/job/allocations.hbs +++ b/ui/app/templates/jobs/job/allocations.hbs @@ -8,8 +8,7 @@ data-test-allocations-search @searchTerm={{mut this.searchTerm}} @onChange={{action this.resetPagination}} - @placeholder="Search allocations..." - /> + @placeholder="Search allocations..." />
@@ -49,69 +48,40 @@ @source={{this.sortedAllocations}} @size={{this.pageSize}} @page={{this.currentPage}} - @class="allocations" as |p| - > + @class="allocations" as |p|> + @class="with-foot" as |t|> - - ID - - - Task Group - - - Created - - - Modified - - - Status - - - Version - - - Client - - - Volume - - - CPU - - - Memory - + ID + Task Group + Created + Modified + Status + Version + Client + Volume + CPU + Memory + @onClick={{action "gotoAllocation" row.model}} />
@@ -119,27 +89,16 @@ {{else}}
-

- No Matches -

-

- No allocations match the term - - {{this.searchTerm}} - -

+

No Matches

+

No allocations match the term {{this.searchTerm}}

{{/if}} {{else}}
-

- No Allocations -

-

- No allocations have been placed. -

+

No Allocations

+

No allocations have been placed.

{{/if}} From ad80c84affcb943829ee5e96c20a3189331ebe1d Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 19:49:05 -0500 Subject: [PATCH 31/40] ui: fix job allocation filter by status, remove version filter, and add tests --- ui/app/controllers/jobs/job/allocations.js | 60 +++----- ui/app/templates/jobs/job/allocations.hbs | 11 +- ui/tests/acceptance/job-allocations-test.js | 147 ++++++++++++++++++++ ui/tests/pages/jobs/job/allocations.js | 7 + 4 files changed, 173 insertions(+), 52 deletions(-) diff --git a/ui/app/controllers/jobs/job/allocations.js b/ui/app/controllers/jobs/job/allocations.js index 2b1861ea8314..d6ec836c47bb 100644 --- a/ui/app/controllers/jobs/job/allocations.js +++ b/ui/app/controllers/jobs/job/allocations.js @@ -12,10 +12,10 @@ import classic from 'ember-classic-decorator'; @classic export default class AllocationsController extends Controller.extend( - Sortable, - Searchable, - WithNamespaceResetting -) { + Sortable, + Searchable, + WithNamespaceResetting + ) { queryParams = [ { currentPage: 'page', @@ -38,15 +38,11 @@ export default class AllocationsController extends Controller.extend( { qpTaskGroup: 'taskGroup', }, - { - qpJobVersion: 'jobVersion', - }, ]; qpStatus = ''; qpClient = ''; qpTaskGroup = ''; - qpJobVersion = ''; currentPage = 1; pageSize = 25; @@ -60,20 +56,16 @@ export default class AllocationsController extends Controller.extend( return ['shortId', 'name', 'taskGroupName']; } - @computed( - 'model.allocations.[]', - 'selectionStatus', - 'selectionClient', - 'selectionTaskGroup', - 'selectionJobVersion' - ) + @computed('model.allocations.[]') get allocations() { - const allocations = this.get('model.allocations') || []; - const { selectionStatus, selectionClient, selectionTaskGroup, selectionJobVersion } = this; + return this.get('model.allocations') || []; + } - if (!allocations.length) return allocations; + @computed('allocations.[]', 'selectionStatus', 'selectionClient', 'selectionTaskGroup') + get filteredAllocations() { + const { selectionStatus, selectionClient, selectionTaskGroup } = this; - return allocations.filter(alloc => { + return this.allocations.filter(alloc => { if (selectionStatus.length && !selectionStatus.includes(alloc.clientStatus)) { return false; } @@ -83,21 +75,17 @@ export default class AllocationsController extends Controller.extend( if (selectionTaskGroup.length && !selectionTaskGroup.includes(alloc.taskGroupName)) { return false; } - if (selectionJobVersion.length && !selectionJobVersion.includes(alloc.jobVersion)) { - return false; - } return true; }); } + @alias('filteredAllocations') listToSort; + @alias('listSorted') listToSearch; + @alias('listSearched') sortedAllocations; + @selection('qpStatus') selectionStatus; @selection('qpClient') selectionClient; @selection('qpTaskGroup') selectionTaskGroup; - @selection('qpJobVersion') selectionJobVersion; - - @alias('allocations') listToSort; - @alias('listSorted') listToSearch; - @alias('listSearched') sortedAllocations; @action gotoAllocation(allocation) { @@ -106,8 +94,7 @@ export default class AllocationsController extends Controller.extend( get optionsAllocationStatus() { return [ - { key: 'queued', label: 'Queued' }, - { key: 'starting', label: 'Starting' }, + { key: 'pending', label: 'Pending' }, { key: 'running', label: 'Running' }, { key: 'complete', label: 'Complete' }, { key: 'failed', label: 'Failed' }, @@ -132,7 +119,7 @@ export default class AllocationsController extends Controller.extend( get optionsTaskGroups() { const taskGroups = Array.from(new Set(this.model.allocations.mapBy('taskGroupName'))).compact(); - // Update query param when the list of clients changes. + // Update query param when the list of task groups changes. scheduleOnce('actions', () => { // eslint-disable-next-line ember/no-side-effects this.set('qpTaskGroup', serialize(intersection(taskGroups, this.selectionTaskGroup))); @@ -141,19 +128,6 @@ export default class AllocationsController extends Controller.extend( return taskGroups.sort().map(tg => ({ key: tg, label: tg })); } - @computed('model.allocations.[]', 'selectionJobVersion') - get optionsJobVersions() { - const jobVersions = Array.from(new Set(this.model.allocations.mapBy('jobVersion'))).compact(); - - // Update query param when the list of clients changes. - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set('qpJobVersion', serialize(intersection(jobVersions, this.selectionJobVersion))); - }); - - return jobVersions.sort().map(jv => ({ key: jv, label: jv })); - } - setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); } diff --git a/ui/app/templates/jobs/job/allocations.hbs b/ui/app/templates/jobs/job/allocations.hbs index 22f3aaac8a0e..2dff6a224fdd 100644 --- a/ui/app/templates/jobs/job/allocations.hbs +++ b/ui/app/templates/jobs/job/allocations.hbs @@ -1,7 +1,7 @@ {{page-title "Job " this.job.name " allocations"}}
- {{#if this.model.allocations.length}} + {{#if this.allocations.length}}
-
diff --git a/ui/tests/acceptance/job-allocations-test.js b/ui/tests/acceptance/job-allocations-test.js index 7c86653c339f..73fcc7207ed4 100644 --- a/ui/tests/acceptance/job-allocations-test.js +++ b/ui/tests/acceptance/job-allocations-test.js @@ -124,4 +124,151 @@ module('Acceptance | job allocations', function(hooks) { assert.ok(Allocations.error.isPresent, 'Error message is shown'); assert.equal(Allocations.error.title, 'Not Found', 'Error message is for 404'); }); + + testFacet('Status', { + facet: Allocations.facets.status, + paramName: 'status', + expectedOptions: ['Pending', 'Running', 'Complete', 'Failed', 'Lost'], + async beforeEach() { + ['pending', 'running', 'complete', 'failed', 'lost'].forEach(s => { + server.createList('allocation', 5, { clientStatus: s }); + }); + await Allocations.visit({ id: job.id }); + }, + filter: (alloc, selection) => alloc.jobId == job.id && selection.includes(alloc.clientStatus), + }); + + testFacet('Client', { + facet: Allocations.facets.client, + paramName: 'client', + expectedOptions(allocs) { + return Array.from( + new Set( + allocs + .filter(alloc => alloc.jobId == job.id) + .mapBy('nodeId') + .map(id => id.split('-')[0]) + ) + ).sort(); + }, + async beforeEach() { + server.createList('node', 5); + server.createList('allocation', 20); + + await Allocations.visit({ id: job.id }); + }, + filter: (alloc, selection) => + alloc.jobId == job.id && selection.includes(alloc.nodeId.split('-')[0]), + }); + + testFacet('Task Group', { + facet: Allocations.facets.taskGroup, + paramName: 'taskGroup', + expectedOptions(allocs) { + return Array.from( + new Set(allocs.filter(alloc => alloc.jobId == job.id).mapBy('taskGroup')) + ).sort(); + }, + async beforeEach() { + job = server.create('job', { + type: 'service', + status: 'running', + groupsCount: 5, + }); + + await Allocations.visit({ id: job.id }); + }, + filter: (alloc, selection) => alloc.jobId == job.id && selection.includes(alloc.taskGroup), + }); }); + +function testFacet(label, { facet, paramName, beforeEach, filter, expectedOptions }) { + test(`facet ${label} | the ${label} facet has the correct options`, async function(assert) { + await beforeEach(); + await facet.toggle(); + + let expectation; + if (typeof expectedOptions === 'function') { + expectation = expectedOptions(server.db.allocations); + } else { + expectation = expectedOptions; + } + + assert.deepEqual( + facet.options.map(option => option.label.trim()), + expectation, + 'Options for facet are as expected' + ); + }); + + test(`facet ${label} | the ${label} facet filters the allocations list by ${label}`, async function(assert) { + let option; + + await beforeEach(); + + await facet.toggle(); + option = facet.options.objectAt(0); + await option.toggle(); + + const selection = [option.key]; + const expectedAllocs = server.db.allocations + .filter(alloc => filter(alloc, selection)) + .sortBy('modifyIndex') + .reverse(); + + Allocations.allocations.forEach((alloc, index) => { + assert.equal( + alloc.id, + expectedAllocs[index].id, + `Allocation at ${index} is ${expectedAllocs[index].id}` + ); + }); + }); + + test(`facet ${label} | selecting multiple options in the ${label} facet results in a broader search`, async function(assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.key); + await option2.toggle(); + selection.push(option2.key); + + const expectedAllocs = server.db.allocations + .filter(alloc => filter(alloc, selection)) + .sortBy('modifyIndex') + .reverse(); + + Allocations.allocations.forEach((alloc, index) => { + assert.equal( + alloc.id, + expectedAllocs[index].id, + `Allocation at ${index} is ${expectedAllocs[index].id}` + ); + }); + }); + + test(`facet ${label} | selecting options in the ${label} facet updates the ${paramName} query param`, async function(assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.key); + await option2.toggle(); + selection.push(option2.key); + + assert.equal( + currentURL(), + `/jobs/${job.id}/allocations?${paramName}=${encodeURIComponent(JSON.stringify(selection))}`, + 'URL has the correct query param key and value' + ); + }); +} diff --git a/ui/tests/pages/jobs/job/allocations.js b/ui/tests/pages/jobs/job/allocations.js index adb48aa704e8..8b15cb932cb0 100644 --- a/ui/tests/pages/jobs/job/allocations.js +++ b/ui/tests/pages/jobs/job/allocations.js @@ -11,6 +11,7 @@ import { import allocations from 'nomad-ui/tests/pages/components/allocations'; import error from 'nomad-ui/tests/pages/components/error'; +import { multiFacet } from 'nomad-ui/tests/pages/components/facet'; export default create({ visit: visitable('/jobs/:id/allocations'), @@ -22,6 +23,12 @@ export default create({ ...allocations(), + facets: { + status: multiFacet('[data-test-allocation-status-facet]'), + client: multiFacet('[data-test-allocation-client-facet]'), + taskGroup: multiFacet('[data-test-allocation-task-group-facet]'), + }, + isEmpty: isPresent('[data-test-empty-allocations-list]'), emptyState: { headline: text('[data-test-empty-allocations-list-headline]'), From e6ee0619c0222aab3ce091d9bd8908803188bfe3 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 20:02:59 -0500 Subject: [PATCH 32/40] ui: fix allocation serializer tests --- ui/tests/unit/serializers/allocation-test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ui/tests/unit/serializers/allocation-test.js b/ui/tests/unit/serializers/allocation-test.js index 80d58bbd6b77..5250d8336a0c 100644 --- a/ui/tests/unit/serializers/allocation-test.js +++ b/ui/tests/unit/serializers/allocation-test.js @@ -35,6 +35,7 @@ module('Unit | Serializer | Allocation', function(hooks) { attributes: { taskGroupName: 'test-group', name: 'test-summary[1]', + namespace: 'test-namespace', modifyTime: sampleDate, createTime: sampleDate, states: [ @@ -102,6 +103,7 @@ module('Unit | Serializer | Allocation', function(hooks) { attributes: { taskGroupName: 'test-group', name: 'test-summary[1]', + namespace: 'test-namespace', modifyTime: sampleDate, createTime: sampleDate, states: [ @@ -172,6 +174,7 @@ module('Unit | Serializer | Allocation', function(hooks) { attributes: { taskGroupName: 'test-group', name: 'test-summary[1]', + namespace: 'test-namespace', modifyTime: sampleDate, createTime: sampleDate, states: [ @@ -259,6 +262,7 @@ module('Unit | Serializer | Allocation', function(hooks) { attributes: { taskGroupName: 'test-group', name: 'test-summary[1]', + namespace: 'test-namespace', modifyTime: sampleDate, createTime: sampleDate, states: [ @@ -332,6 +336,7 @@ module('Unit | Serializer | Allocation', function(hooks) { attributes: { taskGroupName: 'test-group', name: 'test-summary[1]', + namespace: 'test-namespace', modifyTime: sampleDate, createTime: sampleDate, states: [ From efd05eaa54b88a1a1026649c1e2836b64ed5ba70 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 20:23:28 -0500 Subject: [PATCH 33/40] ui: fix volume serializer tests --- ui/tests/unit/serializers/volume-test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/tests/unit/serializers/volume-test.js b/ui/tests/unit/serializers/volume-test.js index 72822cf4950b..792538fc789e 100644 --- a/ui/tests/unit/serializers/volume-test.js +++ b/ui/tests/unit/serializers/volume-test.js @@ -260,6 +260,7 @@ module('Unit | Serializer | Volume', function(hooks) { attributes: { createTime: REF_DATE, modifyTime: REF_DATE, + namespace: 'namespace-2', taskGroupName: 'foobar', wasPreempted: false, states: [], @@ -292,6 +293,7 @@ module('Unit | Serializer | Volume', function(hooks) { attributes: { createTime: REF_DATE, modifyTime: REF_DATE, + namespace: 'namespace-2', taskGroupName: 'write-here', wasPreempted: false, states: [], @@ -324,6 +326,7 @@ module('Unit | Serializer | Volume', function(hooks) { attributes: { createTime: REF_DATE, modifyTime: REF_DATE, + namespace: 'namespace-2', taskGroupName: 'look-if-you-must', wasPreempted: false, states: [], From a8c9676c99c7da42ec71721e061f54cdbc616d76 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 17 Dec 2021 20:41:53 -0500 Subject: [PATCH 34/40] ui: fix action call to set filter query param --- ui/app/controllers/clients/client/index.js | 1 - ui/app/controllers/jobs/job/task-group.js | 1 - ui/app/templates/clients/client/index.hbs | 6 +++--- ui/app/templates/jobs/job/task-group.hbs | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ui/app/controllers/clients/client/index.js b/ui/app/controllers/clients/client/index.js index a918a7b8298b..bc4eeec79b01 100644 --- a/ui/app/controllers/clients/client/index.js +++ b/ui/app/controllers/clients/client/index.js @@ -230,7 +230,6 @@ export default class ClientController extends Controller.extend(Sortable, Search return ns.sort().map(n => ({ key: n, label: n })); } - @action setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); } diff --git a/ui/app/controllers/jobs/job/task-group.js b/ui/app/controllers/jobs/job/task-group.js index 796aab4ce3af..f8a0ec3620f1 100644 --- a/ui/app/controllers/jobs/job/task-group.js +++ b/ui/app/controllers/jobs/job/task-group.js @@ -138,7 +138,6 @@ export default class TaskGroupController extends Controller.extend( return clients.sort().map(dc => ({ key: dc, label: dc })); } - @action setFacetQueryParam(queryParam, selection) { this.set(queryParam, serialize(selection)); } diff --git a/ui/app/templates/clients/client/index.hbs b/ui/app/templates/clients/client/index.hbs index 7e623678c400..c7096c9afa99 100644 --- a/ui/app/templates/clients/client/index.hbs +++ b/ui/app/templates/clients/client/index.hbs @@ -300,21 +300,21 @@ @label="Namespace" @options={{this.optionsNamespace}} @selection={{this.selectionNamespace}} - @onSelect={{action "setFacetQueryParam" "qpNamespace"}} + @onSelect={{action this.setFacetQueryParam "qpNamespace"}} /> Date: Mon, 20 Dec 2021 11:44:21 +0100 Subject: [PATCH 35/40] chore: fixup inconsistent method receiver names. (#11704) --- .golangci.yml | 3 +- client/allocdir/task_dir_nonlinux.go | 2 +- client/allocrunner/csi_hook.go | 4 +- client/allocrunner/state/state.go | 4 +- .../taskrunner/script_check_hook.go | 4 +- client/client.go | 4 +- client/fingerprint_manager.go | 10 +- command/agent/config.go | 16 +-- command/alloc_exec.go | 4 +- command/alloc_fs.go | 4 +- command/alloc_logs.go | 4 +- command/alloc_restart.go | 4 +- command/alloc_signal.go | 4 +- command/alloc_stop.go | 4 +- command/scaling_policy_info.go | 4 +- command/status.go | 2 +- drivers/shared/executor/client.go | 10 +- e2e/e2eutil/e2ejob.go | 4 +- lib/cpuset/cpuset.go | 16 +-- nomad/csi_endpoint.go | 18 +-- nomad/scaling_endpoint.go | 8 +- nomad/state/state_store.go | 8 +- nomad/structs/config/ui.go | 12 +- nomad/structs/config/vault.go | 44 +++--- nomad/structs/csi.go | 10 +- nomad/structs/diff.go | 28 ++-- nomad/structs/structs.go | 128 +++++++++--------- plugins/csi/testing/client.go | 40 +++--- plugins/drivers/testutils/testing.go | 4 +- scheduler/reconcile_util.go | 4 +- 30 files changed, 205 insertions(+), 206 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 4fb6f5e1f942..6be0b4d43cea 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -65,9 +65,8 @@ linters-settings: - commentFormatting - deprecatedComment staticcheck: - # Only enable a single check to start. # I(jrasell) will work on enabling additional checks when possible. - checks: ["ST1020"] + checks: ["ST1020", "ST1016"] issues: exclude: diff --git a/client/allocdir/task_dir_nonlinux.go b/client/allocdir/task_dir_nonlinux.go index 87f813f7feb1..e69bab05689e 100644 --- a/client/allocdir/task_dir_nonlinux.go +++ b/client/allocdir/task_dir_nonlinux.go @@ -4,6 +4,6 @@ package allocdir // currently a noop on non-Linux platforms -func (d *TaskDir) unmountSpecialDirs() error { +func (t *TaskDir) unmountSpecialDirs() error { return nil } diff --git a/client/allocrunner/csi_hook.go b/client/allocrunner/csi_hook.go index da19bf657db6..6504b31cbcb3 100644 --- a/client/allocrunner/csi_hook.go +++ b/client/allocrunner/csi_hook.go @@ -212,8 +212,8 @@ func newCSIHook(ar *allocRunner, logger hclog.Logger, alloc *structs.Allocation, } } -func (h *csiHook) shouldRun() bool { - tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup) +func (c *csiHook) shouldRun() bool { + tg := c.alloc.Job.LookupTaskGroup(c.alloc.TaskGroup) for _, vol := range tg.Volumes { if vol.Type == structs.VolumeTypeCSI { return true diff --git a/client/allocrunner/state/state.go b/client/allocrunner/state/state.go index 6f15e0355e6d..f1316cc57f3c 100644 --- a/client/allocrunner/state/state.go +++ b/client/allocrunner/state/state.go @@ -65,8 +65,8 @@ func (s *State) Copy() *State { } // ClientTerminalStatus returns if the client status is terminal and will no longer transition -func (a *State) ClientTerminalStatus() bool { - switch a.ClientStatus { +func (s *State) ClientTerminalStatus() bool { + switch s.ClientStatus { case structs.AllocClientStatusComplete, structs.AllocClientStatusFailed, structs.AllocClientStatusLost: return true default: diff --git a/client/allocrunner/taskrunner/script_check_hook.go b/client/allocrunner/taskrunner/script_check_hook.go index 65232f70893a..332ec6673d46 100644 --- a/client/allocrunner/taskrunner/script_check_hook.go +++ b/client/allocrunner/taskrunner/script_check_hook.go @@ -399,9 +399,9 @@ const ( // updateTTL updates the state to Consul, performing an exponential backoff // in the case where the check isn't registered in Consul to avoid a race between // service registration and the first check. -func (s *scriptCheck) updateTTL(ctx context.Context, msg, state string) error { +func (sc *scriptCheck) updateTTL(ctx context.Context, msg, state string) error { for attempts := 0; ; attempts++ { - err := s.ttlUpdater.UpdateTTL(s.id, s.consulNamespace, msg, state) + err := sc.ttlUpdater.UpdateTTL(sc.id, sc.consulNamespace, msg, state) if err == nil { return nil } diff --git a/client/client.go b/client/client.go index 8b6d0ad75e2a..1f5025cdc5ab 100644 --- a/client/client.go +++ b/client/client.go @@ -3105,8 +3105,8 @@ func (g *group) Go(f func()) { }() } -func (c *group) AddCh(ch <-chan struct{}) { - c.Go(func() { +func (g *group) AddCh(ch <-chan struct{}) { + g.Go(func() { <-ch }) } diff --git a/client/fingerprint_manager.go b/client/fingerprint_manager.go index 7968b14c7ef1..8a08a077a2c7 100644 --- a/client/fingerprint_manager.go +++ b/client/fingerprint_manager.go @@ -68,16 +68,16 @@ func (fm *FingerprintManager) getNode() *structs.Node { // identifying allowlisted and denylisted fingerprints/drivers. Then, for // those which require periotic checking, it starts a periodic process for // each. -func (fp *FingerprintManager) Run() error { +func (fm *FingerprintManager) Run() error { // First, set up all fingerprints - cfg := fp.getConfig() + cfg := fm.getConfig() // COMPAT(1.0) using inclusive language, whitelist is kept for backward compatibility. allowlistFingerprints := cfg.ReadStringListToMap("fingerprint.allowlist", "fingerprint.whitelist") allowlistFingerprintsEnabled := len(allowlistFingerprints) > 0 // COMPAT(1.0) using inclusive language, blacklist is kept for backward compatibility. denylistFingerprints := cfg.ReadStringListToMap("fingerprint.denylist", "fingerprint.blacklist") - fp.logger.Debug("built-in fingerprints", "fingerprinters", fingerprint.BuiltinFingerprints()) + fm.logger.Debug("built-in fingerprints", "fingerprinters", fingerprint.BuiltinFingerprints()) var availableFingerprints []string var skippedFingerprints []string @@ -96,12 +96,12 @@ func (fp *FingerprintManager) Run() error { availableFingerprints = append(availableFingerprints, name) } - if err := fp.setupFingerprinters(availableFingerprints); err != nil { + if err := fm.setupFingerprinters(availableFingerprints); err != nil { return err } if len(skippedFingerprints) != 0 { - fp.logger.Debug("fingerprint modules skipped due to allow/denylist", + fm.logger.Debug("fingerprint modules skipped due to allow/denylist", "skipped_fingerprinters", skippedFingerprints) } diff --git a/command/agent/config.go b/command/agent/config.go index 2a18f0e2a8a7..740da6cb6b18 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -737,8 +737,8 @@ type Telemetry struct { } // PrefixFilters parses the PrefixFilter field and returns a list of allowed and blocked filters -func (t *Telemetry) PrefixFilters() (allowed, blocked []string, err error) { - for _, rule := range t.PrefixFilter { +func (a *Telemetry) PrefixFilters() (allowed, blocked []string, err error) { + for _, rule := range a.PrefixFilter { if rule == "" { continue } @@ -1448,8 +1448,8 @@ func (a *ACLConfig) Merge(b *ACLConfig) *ACLConfig { } // Merge is used to merge two server configs together -func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig { - result := *a +func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig { + result := *s if b.Enabled { result.Enabled = true @@ -1583,13 +1583,13 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig { result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...) // Copy the start join addresses - result.StartJoin = make([]string, 0, len(a.StartJoin)+len(b.StartJoin)) - result.StartJoin = append(result.StartJoin, a.StartJoin...) + result.StartJoin = make([]string, 0, len(s.StartJoin)+len(b.StartJoin)) + result.StartJoin = append(result.StartJoin, s.StartJoin...) result.StartJoin = append(result.StartJoin, b.StartJoin...) // Copy the retry join addresses - result.RetryJoin = make([]string, 0, len(a.RetryJoin)+len(b.RetryJoin)) - result.RetryJoin = append(result.RetryJoin, a.RetryJoin...) + result.RetryJoin = make([]string, 0, len(s.RetryJoin)+len(b.RetryJoin)) + result.RetryJoin = append(result.RetryJoin, s.RetryJoin...) result.RetryJoin = append(result.RetryJoin, b.RetryJoin...) return &result diff --git a/command/alloc_exec.go b/command/alloc_exec.go index 9555ee03d8af..cf72e033fbd9 100644 --- a/command/alloc_exec.go +++ b/command/alloc_exec.go @@ -70,8 +70,8 @@ func (l *AllocExecCommand) Synopsis() string { return "Execute commands in task" } -func (c *AllocExecCommand) AutocompleteFlags() complete.Flags { - return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), +func (l *AllocExecCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(l.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "--task": complete.PredictAnything, "-job": complete.PredictAnything, diff --git a/command/alloc_fs.go b/command/alloc_fs.go index 9afe28494a35..1912ebc1c7fc 100644 --- a/command/alloc_fs.go +++ b/command/alloc_fs.go @@ -83,8 +83,8 @@ func (f *AllocFSCommand) Synopsis() string { return "Inspect the contents of an allocation directory" } -func (c *AllocFSCommand) AutocompleteFlags() complete.Flags { - return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), +func (f *AllocFSCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(f.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "-H": complete.PredictNothing, "-verbose": complete.PredictNothing, diff --git a/command/alloc_logs.go b/command/alloc_logs.go index 935e7e265f49..24d8229b1db0 100644 --- a/command/alloc_logs.go +++ b/command/alloc_logs.go @@ -75,8 +75,8 @@ func (l *AllocLogsCommand) Synopsis() string { return "Streams the logs of a task." } -func (c *AllocLogsCommand) AutocompleteFlags() complete.Flags { - return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), +func (l *AllocLogsCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(l.Meta.AutocompleteFlags(FlagSetClient), complete.Flags{ "-stderr": complete.PredictNothing, "-verbose": complete.PredictNothing, diff --git a/command/alloc_restart.go b/command/alloc_restart.go index 3aea9538a239..b06f5ccc84b2 100644 --- a/command/alloc_restart.go +++ b/command/alloc_restart.go @@ -13,7 +13,7 @@ type AllocRestartCommand struct { Meta } -func (a *AllocRestartCommand) Help() string { +func (c *AllocRestartCommand) Help() string { helpText := ` Usage: nomad alloc restart [options] @@ -153,7 +153,7 @@ func validateTaskExistsInAllocation(taskName string, alloc *api.Allocation) erro return fmt.Errorf("Could not find task named: %s, found:\n%s", taskName, formatList(foundTaskNames)) } -func (a *AllocRestartCommand) Synopsis() string { +func (c *AllocRestartCommand) Synopsis() string { return "Restart a running allocation" } diff --git a/command/alloc_signal.go b/command/alloc_signal.go index 70b94991223f..e4db386f92f6 100644 --- a/command/alloc_signal.go +++ b/command/alloc_signal.go @@ -13,7 +13,7 @@ type AllocSignalCommand struct { Meta } -func (a *AllocSignalCommand) Help() string { +func (c *AllocSignalCommand) Help() string { helpText := ` Usage: nomad alloc signal [options] @@ -141,7 +141,7 @@ func (c *AllocSignalCommand) Run(args []string) int { return 0 } -func (a *AllocSignalCommand) Synopsis() string { +func (c *AllocSignalCommand) Synopsis() string { return "Signal a running allocation" } diff --git a/command/alloc_stop.go b/command/alloc_stop.go index 3b1d10218ab2..8a130cbaa5ff 100644 --- a/command/alloc_stop.go +++ b/command/alloc_stop.go @@ -11,7 +11,7 @@ type AllocStopCommand struct { Meta } -func (a *AllocStopCommand) Help() string { +func (c *AllocStopCommand) Help() string { helpText := ` Usage: nomad alloc stop [options] Alias: nomad stop @@ -142,6 +142,6 @@ func (c *AllocStopCommand) Run(args []string) int { return mon.monitor(resp.EvalID) } -func (a *AllocStopCommand) Synopsis() string { +func (c *AllocStopCommand) Synopsis() string { return "Stop and reschedule a running allocation" } diff --git a/command/scaling_policy_info.go b/command/scaling_policy_info.go index d255816ae4fe..4e8662055d63 100644 --- a/command/scaling_policy_info.go +++ b/command/scaling_policy_info.go @@ -61,9 +61,9 @@ func (s *ScalingPolicyInfoCommand) AutocompleteFlags() complete.Flags { }) } -func (c *ScalingPolicyInfoCommand) AutocompleteArgs() complete.Predictor { +func (s *ScalingPolicyInfoCommand) AutocompleteArgs() complete.Predictor { return complete.PredictFunc(func(a complete.Args) []string { - client, err := c.Meta.Client() + client, err := s.Meta.Client() if err != nil { return nil } diff --git a/command/status.go b/command/status.go index 54459577c334..e48aefef4c16 100644 --- a/command/status.go +++ b/command/status.go @@ -16,7 +16,7 @@ type StatusCommand struct { verbose bool } -func (s *StatusCommand) Help() string { +func (c *StatusCommand) Help() string { helpText := ` Usage: nomad status [options] diff --git a/drivers/shared/executor/client.go b/drivers/shared/executor/client.go index 44f459e9183a..7ab2dbf6a6ae 100644 --- a/drivers/shared/executor/client.go +++ b/drivers/shared/executor/client.go @@ -195,24 +195,24 @@ func (c *grpcExecutorClient) Exec(deadline time.Time, cmd string, args []string) return resp.Output, int(resp.ExitCode), nil } -func (d *grpcExecutorClient) ExecStreaming(ctx context.Context, +func (c *grpcExecutorClient) ExecStreaming(ctx context.Context, command []string, tty bool, execStream drivers.ExecTaskStream) error { - err := d.execStreaming(ctx, command, tty, execStream) + err := c.execStreaming(ctx, command, tty, execStream) if err != nil { - return grpcutils.HandleGrpcErr(err, d.doneCtx) + return grpcutils.HandleGrpcErr(err, c.doneCtx) } return nil } -func (d *grpcExecutorClient) execStreaming(ctx context.Context, +func (c *grpcExecutorClient) execStreaming(ctx context.Context, command []string, tty bool, execStream drivers.ExecTaskStream) error { - stream, err := d.client.ExecStreaming(ctx) + stream, err := c.client.ExecStreaming(ctx) if err != nil { return err } diff --git a/e2e/e2eutil/e2ejob.go b/e2e/e2eutil/e2ejob.go index 2befe146d2dd..80ecdc1e70cc 100644 --- a/e2e/e2eutil/e2ejob.go +++ b/e2e/e2eutil/e2ejob.go @@ -28,8 +28,8 @@ type e2eJob struct { jobID string } -func (e *e2eJob) Name() string { - return filepath.Base(e.jobfile) +func (j *e2eJob) Name() string { + return filepath.Base(j.jobfile) } // Ensure cluster has leader and at least 1 client node diff --git a/lib/cpuset/cpuset.go b/lib/cpuset/cpuset.go index 7058e46df131..e794d354c5bf 100644 --- a/lib/cpuset/cpuset.go +++ b/lib/cpuset/cpuset.go @@ -102,8 +102,8 @@ func (c CPUSet) Difference(other CPUSet) CPUSet { } // IsSubsetOf returns true if all cpus of the this CPUSet are present in the other CPUSet. -func (s CPUSet) IsSubsetOf(other CPUSet) bool { - for cpu := range s.cpus { +func (c CPUSet) IsSubsetOf(other CPUSet) bool { + for cpu := range c.cpus { if _, ok := other.cpus[cpu]; !ok { return false } @@ -111,9 +111,9 @@ func (s CPUSet) IsSubsetOf(other CPUSet) bool { return true } -func (s CPUSet) IsSupersetOf(other CPUSet) bool { +func (c CPUSet) IsSupersetOf(other CPUSet) bool { for cpu := range other.cpus { - if _, ok := s.cpus[cpu]; !ok { + if _, ok := c.cpus[cpu]; !ok { return false } } @@ -121,9 +121,9 @@ func (s CPUSet) IsSupersetOf(other CPUSet) bool { } // ContainsAny returns true if any cpus in other CPUSet are present -func (s CPUSet) ContainsAny(other CPUSet) bool { +func (c CPUSet) ContainsAny(other CPUSet) bool { for cpu := range other.cpus { - if _, ok := s.cpus[cpu]; ok { + if _, ok := c.cpus[cpu]; ok { return true } } @@ -131,8 +131,8 @@ func (s CPUSet) ContainsAny(other CPUSet) bool { } // Equals tests the equality of the elements in the CPUSet -func (s CPUSet) Equals(other CPUSet) bool { - return reflect.DeepEqual(s.cpus, other.cpus) +func (c CPUSet) Equals(other CPUSet) bool { + return reflect.DeepEqual(c.cpus, other.cpus) } // Parse parses the Linux cpuset format into a CPUSet diff --git a/nomad/csi_endpoint.go b/nomad/csi_endpoint.go index eb6954eed2f1..825abf4be21a 100644 --- a/nomad/csi_endpoint.go +++ b/nomad/csi_endpoint.go @@ -23,9 +23,9 @@ type CSIVolume struct { // QueryACLObj looks up the ACL token in the request and returns the acl.ACL object // - fallback to node secret ids -func (srv *Server) QueryACLObj(args *structs.QueryOptions, allowNodeAccess bool) (*acl.ACL, error) { +func (s *Server) QueryACLObj(args *structs.QueryOptions, allowNodeAccess bool) (*acl.ACL, error) { // Lookup the token - aclObj, err := srv.ResolveToken(args.AuthToken) + aclObj, err := s.ResolveToken(args.AuthToken) if err != nil { // If ResolveToken had an unexpected error return that if !structs.IsErrTokenNotFound(err) { @@ -41,7 +41,7 @@ func (srv *Server) QueryACLObj(args *structs.QueryOptions, allowNodeAccess bool) ws := memdb.NewWatchSet() // Attempt to lookup AuthToken as a Node.SecretID since nodes may call // call this endpoint and don't have an ACL token. - node, stateErr := srv.fsm.State().NodeBySecretID(ws, args.AuthToken) + node, stateErr := s.fsm.State().NodeBySecretID(ws, args.AuthToken) if stateErr != nil { // Return the original ResolveToken error with this err var merr multierror.Error @@ -60,13 +60,13 @@ func (srv *Server) QueryACLObj(args *structs.QueryOptions, allowNodeAccess bool) } // WriteACLObj calls QueryACLObj for a WriteRequest -func (srv *Server) WriteACLObj(args *structs.WriteRequest, allowNodeAccess bool) (*acl.ACL, error) { +func (s *Server) WriteACLObj(args *structs.WriteRequest, allowNodeAccess bool) (*acl.ACL, error) { opts := &structs.QueryOptions{ Region: args.RequestRegion(), Namespace: args.RequestNamespace(), AuthToken: args.AuthToken, } - return srv.QueryACLObj(opts, allowNodeAccess) + return s.QueryACLObj(opts, allowNodeAccess) } const ( @@ -75,17 +75,17 @@ const ( ) // replySetIndex sets the reply with the last index that modified the table -func (srv *Server) replySetIndex(table string, reply *structs.QueryMeta) error { - s := srv.fsm.State() +func (s *Server) replySetIndex(table string, reply *structs.QueryMeta) error { + fmsState := s.fsm.State() - index, err := s.Index(table) + index, err := fmsState.Index(table) if err != nil { return err } reply.Index = index // Set the query response - srv.setQueryMeta(reply) + s.setQueryMeta(reply) return nil } diff --git a/nomad/scaling_endpoint.go b/nomad/scaling_endpoint.go index bb87769a234b..dd8c5fe11993 100644 --- a/nomad/scaling_endpoint.go +++ b/nomad/scaling_endpoint.go @@ -137,9 +137,9 @@ func (p *Scaling) GetPolicy(args *structs.ScalingPolicySpecificRequest, return p.srv.blockingRPC(&opts) } -func (j *Scaling) listAllNamespaces(args *structs.ScalingPolicyListRequest, reply *structs.ScalingPolicyListResponse) error { +func (p *Scaling) listAllNamespaces(args *structs.ScalingPolicyListRequest, reply *structs.ScalingPolicyListResponse) error { // Check for list-job permissions - aclObj, err := j.srv.ResolveToken(args.AuthToken) + aclObj, err := p.srv.ResolveToken(args.AuthToken) if err != nil { return err } @@ -197,8 +197,8 @@ func (j *Scaling) listAllNamespaces(args *structs.ScalingPolicyListRequest, repl reply.Index = helper.Uint64Max(1, index) // Set the query response - j.srv.setQueryMeta(&reply.QueryMeta) + p.srv.setQueryMeta(&reply.QueryMeta) return nil }} - return j.srv.blockingRPC(&opts) + return p.srv.blockingRPC(&opts) } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 81cbd7c633c8..93a7f27d642b 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -6270,13 +6270,13 @@ type StateRestore struct { } // Abort is used to abort the restore operation -func (s *StateRestore) Abort() { - s.txn.Abort() +func (r *StateRestore) Abort() { + r.txn.Abort() } // Commit is used to commit the restore operation -func (s *StateRestore) Commit() error { - return s.txn.Commit() +func (r *StateRestore) Commit() error { + return r.txn.Commit() } // NodeRestore is used to restore a node diff --git a/nomad/structs/config/ui.go b/nomad/structs/config/ui.go index 7577fa6d9fd7..1626544fb4a5 100644 --- a/nomad/structs/config/ui.go +++ b/nomad/structs/config/ui.go @@ -60,8 +60,8 @@ func (old *UIConfig) Copy() *UIConfig { // Merge returns a new UI configuration by merging another UI // configuration into this one -func (this *UIConfig) Merge(other *UIConfig) *UIConfig { - result := this.Copy() +func (old *UIConfig) Merge(other *UIConfig) *UIConfig { + result := old.Copy() if other == nil { return result } @@ -86,8 +86,8 @@ func (old *ConsulUIConfig) Copy() *ConsulUIConfig { // Merge returns a new Consul UI configuration by merging another Consul UI // configuration into this one -func (this *ConsulUIConfig) Merge(other *ConsulUIConfig) *ConsulUIConfig { - result := this.Copy() +func (old *ConsulUIConfig) Merge(other *ConsulUIConfig) *ConsulUIConfig { + result := old.Copy() if result == nil { result = &ConsulUIConfig{} } @@ -114,8 +114,8 @@ func (old *VaultUIConfig) Copy() *VaultUIConfig { // Merge returns a new Vault UI configuration by merging another Vault UI // configuration into this one -func (this *VaultUIConfig) Merge(other *VaultUIConfig) *VaultUIConfig { - result := this.Copy() +func (old *VaultUIConfig) Merge(other *VaultUIConfig) *VaultUIConfig { + result := old.Copy() if result == nil { result = &VaultUIConfig{} } diff --git a/nomad/structs/config/vault.go b/nomad/structs/config/vault.go index 2c7921ea5a56..83a239a19ce0 100644 --- a/nomad/structs/config/vault.go +++ b/nomad/structs/config/vault.go @@ -92,19 +92,19 @@ func DefaultVaultConfig() *VaultConfig { } // IsEnabled returns whether the config enables Vault integration -func (a *VaultConfig) IsEnabled() bool { - return a.Enabled != nil && *a.Enabled +func (c *VaultConfig) IsEnabled() bool { + return c.Enabled != nil && *c.Enabled } // AllowsUnauthenticated returns whether the config allows unauthenticated // access to Vault -func (a *VaultConfig) AllowsUnauthenticated() bool { - return a.AllowUnauthenticated != nil && *a.AllowUnauthenticated +func (c *VaultConfig) AllowsUnauthenticated() bool { + return c.AllowUnauthenticated != nil && *c.AllowUnauthenticated } // Merge merges two Vault configurations together. -func (a *VaultConfig) Merge(b *VaultConfig) *VaultConfig { - result := *a +func (c *VaultConfig) Merge(b *VaultConfig) *VaultConfig { + result := *c if b.Token != "" { result.Token = b.Token @@ -190,51 +190,51 @@ func (c *VaultConfig) Copy() *VaultConfig { // IsEqual compares two Vault configurations and returns a boolean indicating // if they are equal. -func (a *VaultConfig) IsEqual(b *VaultConfig) bool { - if a == nil && b != nil { +func (c *VaultConfig) IsEqual(b *VaultConfig) bool { + if c == nil && b != nil { return false } - if a != nil && b == nil { + if c != nil && b == nil { return false } - if a.Token != b.Token { + if c.Token != b.Token { return false } - if a.Role != b.Role { + if c.Role != b.Role { return false } - if a.TaskTokenTTL != b.TaskTokenTTL { + if c.TaskTokenTTL != b.TaskTokenTTL { return false } - if a.Addr != b.Addr { + if c.Addr != b.Addr { return false } - if a.ConnectionRetryIntv.Nanoseconds() != b.ConnectionRetryIntv.Nanoseconds() { + if c.ConnectionRetryIntv.Nanoseconds() != b.ConnectionRetryIntv.Nanoseconds() { return false } - if a.TLSCaFile != b.TLSCaFile { + if c.TLSCaFile != b.TLSCaFile { return false } - if a.TLSCaPath != b.TLSCaPath { + if c.TLSCaPath != b.TLSCaPath { return false } - if a.TLSCertFile != b.TLSCertFile { + if c.TLSCertFile != b.TLSCertFile { return false } - if a.TLSKeyFile != b.TLSKeyFile { + if c.TLSKeyFile != b.TLSKeyFile { return false } - if a.TLSServerName != b.TLSServerName { + if c.TLSServerName != b.TLSServerName { return false } - if a.AllowUnauthenticated != b.AllowUnauthenticated { + if c.AllowUnauthenticated != b.AllowUnauthenticated { return false } - if a.TLSSkipVerify != b.TLSSkipVerify { + if c.TLSSkipVerify != b.TLSSkipVerify { return false } - if a.Enabled != b.Enabled { + if c.Enabled != b.Enabled { return false } return true diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index 539501301258..1ae51d87bbbf 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -185,17 +185,17 @@ func (o *CSIMountOptions) Merge(p *CSIMountOptions) { var _ fmt.Stringer = &CSIMountOptions{} var _ fmt.GoStringer = &CSIMountOptions{} -func (v *CSIMountOptions) String() string { +func (o *CSIMountOptions) String() string { mountFlagsString := "nil" - if len(v.MountFlags) != 0 { + if len(o.MountFlags) != 0 { mountFlagsString = "[REDACTED]" } - return fmt.Sprintf("csi.CSIOptions(FSType: %s, MountFlags: %s)", v.FSType, mountFlagsString) + return fmt.Sprintf("csi.CSIOptions(FSType: %s, MountFlags: %s)", o.FSType, mountFlagsString) } -func (v *CSIMountOptions) GoString() string { - return v.String() +func (o *CSIMountOptions) GoString() string { + return o.String() } // CSISecrets contain optional additional configuration that can be used diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index d0f9e5bb2272..fe8a0013540d 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -1901,24 +1901,24 @@ func (r *Resources) Diff(other *Resources, contextual bool) *ObjectDiff { // Diff returns a diff of two network resources. If contextual diff is enabled, // non-changed fields will still be returned. -func (r *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectDiff { +func (n *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectDiff { diff := &ObjectDiff{Type: DiffTypeNone, Name: "Network"} var oldPrimitiveFlat, newPrimitiveFlat map[string]string filter := []string{"Device", "CIDR", "IP"} - if reflect.DeepEqual(r, other) { + if reflect.DeepEqual(n, other) { return nil - } else if r == nil { - r = &NetworkResource{} + } else if n == nil { + n = &NetworkResource{} diff.Type = DiffTypeAdded newPrimitiveFlat = flatmap.Flatten(other, filter, true) } else if other == nil { other = &NetworkResource{} diff.Type = DiffTypeDeleted - oldPrimitiveFlat = flatmap.Flatten(r, filter, true) + oldPrimitiveFlat = flatmap.Flatten(n, filter, true) } else { diff.Type = DiffTypeEdited - oldPrimitiveFlat = flatmap.Flatten(r, filter, true) + oldPrimitiveFlat = flatmap.Flatten(n, filter, true) newPrimitiveFlat = flatmap.Flatten(other, filter, true) } @@ -1926,8 +1926,8 @@ func (r *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectD diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) // Port diffs - resPorts := portDiffs(r.ReservedPorts, other.ReservedPorts, false, contextual) - dynPorts := portDiffs(r.DynamicPorts, other.DynamicPorts, true, contextual) + resPorts := portDiffs(n.ReservedPorts, other.ReservedPorts, false, contextual) + dynPorts := portDiffs(n.DynamicPorts, other.DynamicPorts, true, contextual) if resPorts != nil { diff.Objects = append(diff.Objects, resPorts...) } @@ -1935,7 +1935,7 @@ func (r *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectD diff.Objects = append(diff.Objects, dynPorts...) } - if dnsDiff := r.DNS.Diff(other.DNS, contextual); dnsDiff != nil { + if dnsDiff := n.DNS.Diff(other.DNS, contextual); dnsDiff != nil { diff.Objects = append(diff.Objects, dnsDiff) } @@ -1943,8 +1943,8 @@ func (r *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectD } // Diff returns a diff of two DNSConfig structs -func (c *DNSConfig) Diff(other *DNSConfig, contextual bool) *ObjectDiff { - if reflect.DeepEqual(c, other) { +func (d *DNSConfig) Diff(other *DNSConfig, contextual bool) *ObjectDiff { + if reflect.DeepEqual(d, other) { return nil } @@ -1964,15 +1964,15 @@ func (c *DNSConfig) Diff(other *DNSConfig, contextual bool) *ObjectDiff { diff := &ObjectDiff{Type: DiffTypeNone, Name: "DNS"} var oldPrimitiveFlat, newPrimitiveFlat map[string]string - if c == nil { + if d == nil { diff.Type = DiffTypeAdded newPrimitiveFlat = flatten(other) } else if other == nil { diff.Type = DiffTypeDeleted - oldPrimitiveFlat = flatten(c) + oldPrimitiveFlat = flatten(d) } else { diff.Type = DiffTypeEdited - oldPrimitiveFlat = flatten(c) + oldPrimitiveFlat = flatten(d) newPrimitiveFlat = flatten(other) } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 742fce06db69..b856b9ddb6c4 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -2609,23 +2609,23 @@ type NetworkResource struct { DynamicPorts []Port // Host Dynamically assigned ports } -func (nr *NetworkResource) Hash() uint32 { +func (n *NetworkResource) Hash() uint32 { var data []byte - data = append(data, []byte(fmt.Sprintf("%s%s%s%s%s%d", nr.Mode, nr.Device, nr.CIDR, nr.IP, nr.Hostname, nr.MBits))...) + data = append(data, []byte(fmt.Sprintf("%s%s%s%s%s%d", n.Mode, n.Device, n.CIDR, n.IP, n.Hostname, n.MBits))...) - for i, port := range nr.ReservedPorts { + for i, port := range n.ReservedPorts { data = append(data, []byte(fmt.Sprintf("r%d%s%d%d", i, port.Label, port.Value, port.To))...) } - for i, port := range nr.DynamicPorts { + for i, port := range n.DynamicPorts { data = append(data, []byte(fmt.Sprintf("d%d%s%d%d", i, port.Label, port.Value, port.To))...) } return crc32.ChecksumIEEE(data) } -func (nr *NetworkResource) Equals(other *NetworkResource) bool { - return nr.Hash() == other.Hash() +func (n *NetworkResource) Equals(other *NetworkResource) bool { + return n.Hash() == other.Hash() } func (n *NetworkResource) Canonicalize() { @@ -3182,12 +3182,12 @@ type DeviceIdTuple struct { Name string } -func (d *DeviceIdTuple) String() string { - if d == nil { +func (id *DeviceIdTuple) String() string { + if id == nil { return "" } - return fmt.Sprintf("%s/%s/%s", d.Vendor, d.Type, d.Name) + return fmt.Sprintf("%s/%s/%s", id.Vendor, id.Type, id.Name) } // Matches returns if this Device ID is a superset of the passed ID. @@ -7852,98 +7852,98 @@ type TaskEvent struct { GenericSource string } -func (event *TaskEvent) PopulateEventDisplayMessage() { +func (e *TaskEvent) PopulateEventDisplayMessage() { // Build up the description based on the event type. - if event == nil { //TODO(preetha) needs investigation alloc_runner's Run method sends a nil event when sigterming nomad. Why? + if e == nil { //TODO(preetha) needs investigation alloc_runner's Run method sends a nil event when sigterming nomad. Why? return } - if event.DisplayMessage != "" { + if e.DisplayMessage != "" { return } var desc string - switch event.Type { + switch e.Type { case TaskSetup: - desc = event.Message + desc = e.Message case TaskStarted: desc = "Task started by client" case TaskReceived: desc = "Task received by client" case TaskFailedValidation: - if event.ValidationError != "" { - desc = event.ValidationError + if e.ValidationError != "" { + desc = e.ValidationError } else { desc = "Validation of task failed" } case TaskSetupFailure: - if event.SetupError != "" { - desc = event.SetupError + if e.SetupError != "" { + desc = e.SetupError } else { desc = "Task setup failed" } case TaskDriverFailure: - if event.DriverError != "" { - desc = event.DriverError + if e.DriverError != "" { + desc = e.DriverError } else { desc = "Failed to start task" } case TaskDownloadingArtifacts: desc = "Client is downloading artifacts" case TaskArtifactDownloadFailed: - if event.DownloadError != "" { - desc = event.DownloadError + if e.DownloadError != "" { + desc = e.DownloadError } else { desc = "Failed to download artifacts" } case TaskKilling: - if event.KillReason != "" { - desc = event.KillReason - } else if event.KillTimeout != 0 { - desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) + if e.KillReason != "" { + desc = e.KillReason + } else if e.KillTimeout != 0 { + desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", e.KillTimeout) } else { desc = "Sent interrupt" } case TaskKilled: - if event.KillError != "" { - desc = event.KillError + if e.KillError != "" { + desc = e.KillError } else { desc = "Task successfully killed" } case TaskTerminated: var parts []string - parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) + parts = append(parts, fmt.Sprintf("Exit Code: %d", e.ExitCode)) - if event.Signal != 0 { - parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) + if e.Signal != 0 { + parts = append(parts, fmt.Sprintf("Signal: %d", e.Signal)) } - if event.Message != "" { - parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) + if e.Message != "" { + parts = append(parts, fmt.Sprintf("Exit Message: %q", e.Message)) } desc = strings.Join(parts, ", ") case TaskRestarting: - in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) - if event.RestartReason != "" && event.RestartReason != ReasonWithinPolicy { - desc = fmt.Sprintf("%s - %s", event.RestartReason, in) + in := fmt.Sprintf("Task restarting in %v", time.Duration(e.StartDelay)) + if e.RestartReason != "" && e.RestartReason != ReasonWithinPolicy { + desc = fmt.Sprintf("%s - %s", e.RestartReason, in) } else { desc = in } case TaskNotRestarting: - if event.RestartReason != "" { - desc = event.RestartReason + if e.RestartReason != "" { + desc = e.RestartReason } else { desc = "Task exceeded restart policy" } case TaskSiblingFailed: - if event.FailedSibling != "" { - desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling) + if e.FailedSibling != "" { + desc = fmt.Sprintf("Task's sibling %q failed", e.FailedSibling) } else { desc = "Task's sibling failed" } case TaskSignaling: - sig := event.TaskSignal - reason := event.TaskSignalReason + sig := e.TaskSignal + reason := e.TaskSignalReason if sig == "" && reason == "" { desc = "Task being sent a signal" @@ -7955,47 +7955,47 @@ func (event *TaskEvent) PopulateEventDisplayMessage() { desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason) } case TaskRestartSignal: - if event.RestartReason != "" { - desc = event.RestartReason + if e.RestartReason != "" { + desc = e.RestartReason } else { desc = "Task signaled to restart" } case TaskDriverMessage: - desc = event.DriverMessage + desc = e.DriverMessage case TaskLeaderDead: desc = "Leader Task in Group dead" case TaskMainDead: desc = "Main tasks in the group died" default: - desc = event.Message + desc = e.Message } - event.DisplayMessage = desc + e.DisplayMessage = desc } -func (te *TaskEvent) GoString() string { - return fmt.Sprintf("%v - %v", te.Time, te.Type) +func (e *TaskEvent) GoString() string { + return fmt.Sprintf("%v - %v", e.Time, e.Type) } // SetDisplayMessage sets the display message of TaskEvent -func (te *TaskEvent) SetDisplayMessage(msg string) *TaskEvent { - te.DisplayMessage = msg - return te +func (e *TaskEvent) SetDisplayMessage(msg string) *TaskEvent { + e.DisplayMessage = msg + return e } // SetMessage sets the message of TaskEvent -func (te *TaskEvent) SetMessage(msg string) *TaskEvent { - te.Message = msg - te.Details["message"] = msg - return te +func (e *TaskEvent) SetMessage(msg string) *TaskEvent { + e.Message = msg + e.Details["message"] = msg + return e } -func (te *TaskEvent) Copy() *TaskEvent { - if te == nil { +func (e *TaskEvent) Copy() *TaskEvent { + if e == nil { return nil } copy := new(TaskEvent) - *copy = *te + *copy = *e return copy } @@ -11095,7 +11095,7 @@ type ACLPolicy struct { } // SetHash is used to compute and set the hash of the ACL policy -func (c *ACLPolicy) SetHash() []byte { +func (a *ACLPolicy) SetHash() []byte { // Initialize a 256bit Blake2 hash (32 bytes) hash, err := blake2b.New256(nil) if err != nil { @@ -11103,15 +11103,15 @@ func (c *ACLPolicy) SetHash() []byte { } // Write all the user set fields - _, _ = hash.Write([]byte(c.Name)) - _, _ = hash.Write([]byte(c.Description)) - _, _ = hash.Write([]byte(c.Rules)) + _, _ = hash.Write([]byte(a.Name)) + _, _ = hash.Write([]byte(a.Description)) + _, _ = hash.Write([]byte(a.Rules)) // Finalize the hash hashVal := hash.Sum(nil) // Set and return the hash - c.Hash = hashVal + a.Hash = hashVal return hashVal } diff --git a/plugins/csi/testing/client.go b/plugins/csi/testing/client.go index 08168625b8f3..dc180d058205 100644 --- a/plugins/csi/testing/client.go +++ b/plugins/csi/testing/client.go @@ -63,18 +63,18 @@ func NewControllerClient() *ControllerClient { return &ControllerClient{} } -func (f *ControllerClient) Reset() { - f.NextErr = nil - f.NextCapabilitiesResponse = nil - f.NextPublishVolumeResponse = nil - f.NextUnpublishVolumeResponse = nil - f.NextValidateVolumeCapabilitiesResponse = nil - f.NextCreateVolumeResponse = nil - f.NextDeleteVolumeResponse = nil - f.NextListVolumesResponse = nil - f.NextCreateSnapshotResponse = nil - f.NextDeleteSnapshotResponse = nil - f.NextListSnapshotsResponse = nil +func (c *ControllerClient) Reset() { + c.NextErr = nil + c.NextCapabilitiesResponse = nil + c.NextPublishVolumeResponse = nil + c.NextUnpublishVolumeResponse = nil + c.NextValidateVolumeCapabilitiesResponse = nil + c.NextCreateVolumeResponse = nil + c.NextDeleteVolumeResponse = nil + c.NextListVolumesResponse = nil + c.NextCreateSnapshotResponse = nil + c.NextDeleteSnapshotResponse = nil + c.NextListSnapshotsResponse = nil } func (c *ControllerClient) ControllerGetCapabilities(ctx context.Context, in *csipbv1.ControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*csipbv1.ControllerGetCapabilitiesResponse, error) { @@ -144,14 +144,14 @@ func NewNodeClient() *NodeClient { return &NodeClient{} } -func (f *NodeClient) Reset() { - f.NextErr = nil - f.NextCapabilitiesResponse = nil - f.NextGetInfoResponse = nil - f.NextStageVolumeResponse = nil - f.NextUnstageVolumeResponse = nil - f.NextPublishVolumeResponse = nil - f.NextUnpublishVolumeResponse = nil +func (c *NodeClient) Reset() { + c.NextErr = nil + c.NextCapabilitiesResponse = nil + c.NextGetInfoResponse = nil + c.NextStageVolumeResponse = nil + c.NextUnstageVolumeResponse = nil + c.NextPublishVolumeResponse = nil + c.NextUnpublishVolumeResponse = nil } func (c *NodeClient) NodeGetCapabilities(ctx context.Context, in *csipbv1.NodeGetCapabilitiesRequest, opts ...grpc.CallOption) (*csipbv1.NodeGetCapabilitiesResponse, error) { diff --git a/plugins/drivers/testutils/testing.go b/plugins/drivers/testutils/testing.go index 32cc24477315..d8fd2f75d459 100644 --- a/plugins/drivers/testutils/testing.go +++ b/plugins/drivers/testutils/testing.go @@ -36,8 +36,8 @@ type DriverHarness struct { impl drivers.DriverPlugin } -func (d *DriverHarness) Impl() drivers.DriverPlugin { - return d.impl +func (h *DriverHarness) Impl() drivers.DriverPlugin { + return h.impl } func NewDriverHarness(t testing.T, d drivers.DriverPlugin) *DriverHarness { logger := testlog.HCLogger(t).Named("driver_harness") diff --git a/scheduler/reconcile_util.go b/scheduler/reconcile_util.go index 8ee080a0cc1b..40ced8a864bd 100644 --- a/scheduler/reconcile_util.go +++ b/scheduler/reconcile_util.go @@ -394,9 +394,9 @@ func (a allocSet) filterByDeployment(id string) (match, nonmatch allocSet) { // delayByStopAfterClientDisconnect returns a delay for any lost allocation that's got a // stop_after_client_disconnect configured -func (as allocSet) delayByStopAfterClientDisconnect() (later []*delayedRescheduleInfo) { +func (a allocSet) delayByStopAfterClientDisconnect() (later []*delayedRescheduleInfo) { now := time.Now().UTC() - for _, a := range as { + for _, a := range a { if !a.ShouldClientStop() { continue } From 3740c24d9e71cb7e19ace46cd742390ae0b0c80b Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 20 Dec 2021 12:23:50 -0500 Subject: [PATCH 36/40] api: respect wildcard in evaluations list API (#11710) --- .changelog/11710.txt | 3 ++ nomad/eval_endpoint.go | 4 ++- nomad/eval_endpoint_test.go | 35 ++++++++++++++++++++++++ website/content/api-docs/evaluations.mdx | 4 +++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 .changelog/11710.txt diff --git a/.changelog/11710.txt b/.changelog/11710.txt new file mode 100644 index 000000000000..ac405d372c3f --- /dev/null +++ b/.changelog/11710.txt @@ -0,0 +1,3 @@ +```release-note:improvement +api: Updated the evaluations list API to respect wildcard namespaces +``` diff --git a/nomad/eval_endpoint.go b/nomad/eval_endpoint.go index e3a14946d89e..5fe4d3658dee 100644 --- a/nomad/eval_endpoint.go +++ b/nomad/eval_endpoint.go @@ -353,7 +353,9 @@ func (e *Eval) List(args *structs.EvalListRequest, // Scan all the evaluations var err error var iter memdb.ResultIterator - if prefix := args.QueryOptions.Prefix; prefix != "" { + if args.RequestNamespace() == structs.AllNamespacesSentinel { + iter, err = store.Evals(ws) + } else if prefix := args.QueryOptions.Prefix; prefix != "" { iter, err = store.EvalsByIDPrefix(ws, args.RequestNamespace(), prefix) } else { iter, err = store.EvalsByNamespace(ws, args.RequestNamespace()) diff --git a/nomad/eval_endpoint_test.go b/nomad/eval_endpoint_test.go index ef142a9d9d6a..2b78c6b72d79 100644 --- a/nomad/eval_endpoint_test.go +++ b/nomad/eval_endpoint_test.go @@ -718,6 +718,41 @@ func TestEvalEndpoint_List(t *testing.T) { } +func TestEvalEndpoint_ListAllNamespaces(t *testing.T) { + t.Parallel() + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + eval1 := mock.Eval() + eval1.ID = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" + eval2 := mock.Eval() + eval2.ID = "aaaabbbb-3350-4b4b-d185-0e1992ed43e9" + s1.fsm.State().UpsertEvals(structs.MsgTypeTestSetup, 1000, []*structs.Evaluation{eval1, eval2}) + + // Lookup the eval + get := &structs.EvalListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: "*", + }, + } + var resp structs.EvalListResponse + if err := msgpackrpc.CallWithCodec(codec, "Eval.List", get, &resp); err != nil { + t.Fatalf("err: %v", err) + } + if resp.Index != 1000 { + t.Fatalf("Bad index: %d %d", resp.Index, 1000) + } + + if len(resp.Evaluations) != 2 { + t.Fatalf("bad: %#v", resp.Evaluations) + } +} + func TestEvalEndpoint_List_ACL(t *testing.T) { t.Parallel() diff --git a/website/content/api-docs/evaluations.mdx b/website/content/api-docs/evaluations.mdx index 9711140edd17..ab78127eaeab 100644 --- a/website/content/api-docs/evaluations.mdx +++ b/website/content/api-docs/evaluations.mdx @@ -49,6 +49,10 @@ The table below shows this endpoint's support for specific evaluation status (one of `blocked`, `pending`, `complete`, `failed`, or `canceled`). +- `namespace` `(string: "default")` - Specifies the target + namespace. Specifying `*` will return all evaluations across all + authorized namespaces. + ### Sample Request ```shell-session From 84ef826d1cc714dc217d8ef6af0a43e549ec394e Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Mon, 20 Dec 2021 13:45:20 -0500 Subject: [PATCH 37/40] changelog: add entries for #11555, #11557, and #11687 (#11706) --- .changelog/11555.txt | 3 +++ .changelog/11557.txt | 3 +++ .changelog/11687.txt | 3 +++ 3 files changed, 9 insertions(+) create mode 100644 .changelog/11555.txt create mode 100644 .changelog/11557.txt create mode 100644 .changelog/11687.txt diff --git a/.changelog/11555.txt b/.changelog/11555.txt new file mode 100644 index 000000000000..f6bd6de2e1ea --- /dev/null +++ b/.changelog/11555.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: Added `ui` configuration block +``` diff --git a/.changelog/11557.txt b/.changelog/11557.txt new file mode 100644 index 000000000000..abed090353b9 --- /dev/null +++ b/.changelog/11557.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: Display the Consul and Vault links configured in the agent +``` diff --git a/.changelog/11687.txt b/.changelog/11687.txt new file mode 100644 index 000000000000..e61a16efbb10 --- /dev/null +++ b/.changelog/11687.txt @@ -0,0 +1,3 @@ +```release-note:improvement +ui: Display section title in the navigation breadcrumbs +``` From 649f1ab6df329b168e920c14db486242784c3035 Mon Sep 17 00:00:00 2001 From: Guilherme <3874515+gjpin@users.noreply.github.com> Date: Mon, 20 Dec 2021 22:09:15 +0000 Subject: [PATCH 38/40] Fix 'check calculations' link (#11420) --- website/content/tools/autoscaling/internals/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/tools/autoscaling/internals/index.mdx b/website/content/tools/autoscaling/internals/index.mdx index efd07ce958ab..b7844e129449 100644 --- a/website/content/tools/autoscaling/internals/index.mdx +++ b/website/content/tools/autoscaling/internals/index.mdx @@ -12,4 +12,4 @@ This section covers the internals of the Nomad Autoscaler and explains the technical details of how it functions, its architecture, and sub-systems. - [Autoscaler plugins](/tools/autoscaling/internals/plugins) -- [Check calculations](/tools/autoscaling/interals/checks) +- [Check calculations](/tools/autoscaling/internals/checks) From 20bbdba0416cdcfed5e9399d50ed47ba21e41e83 Mon Sep 17 00:00:00 2001 From: Andy Assareh Date: Mon, 20 Dec 2021 14:10:44 -0800 Subject: [PATCH 39/40] Mesh Gateway doc enhancements (#11354) * Mesh Gateway doc enhancements 1. I believe this line should be corrected to add mesh as one of the choices 2. I found that we are not setting this meta, and it is a required element for wan federation. I believe it would be helpful and potentially time saving to note that right here. --- website/content/docs/job-specification/gateway.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/website/content/docs/job-specification/gateway.mdx b/website/content/docs/job-specification/gateway.mdx index 0987b6ced19b..50641bbd6bde 100644 --- a/website/content/docs/job-specification/gateway.mdx +++ b/website/content/docs/job-specification/gateway.mdx @@ -163,7 +163,11 @@ envoy_gateway_bind_addresses "" { ### `mesh` Parameters - The `mesh` block currently does not have any configurable parameters. +The `mesh` block currently does not have any configurable parameters. + +~> **Note:** If using the Mesh Gateway for [WAN Federation][connect_mesh_gw], +the additional piece of service metadata `{"consul-wan-federation":"1"}` must +be applied. This can be done with the service [`meta`][meta] parameter. ### Gateway with host networking @@ -630,6 +634,7 @@ job "countdash-mesh-two" { [address]: /docs/job-specification/gateway#address-parameters [advanced configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration [connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout +[connect_mesh_gw]: https://www.consul.io/docs/connect/gateways/mesh-gateway/wan-federation-via-mesh-gateways#mesh-gateways [envoy docker]: https://hub.docker.com/r/envoyproxy/envoy/tags [ingress]: /docs/job-specification/gateway#ingress-parameters [proxy]: /docs/job-specification/gateway#proxy-parameters @@ -641,3 +646,4 @@ job "countdash-mesh-two" { [terminating]: /docs/job-specification/gateway#terminating-parameters [tls]: /docs/job-specification/gateway#tls-parameters [mesh]: /docs/job-specification/gateway#mesh-parameters +[meta]: /docs/job-specification/service#meta From 2d4e5b8fe93cda41398ed054a5f17eca2ecc7446 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Tue, 21 Dec 2021 10:10:01 -0500 Subject: [PATCH 40/40] scheduler: fix quadratic performance with spread blocks (#11712) When the scheduler picks a node for each evaluation, the `LimitIterator` provides at most 2 eligible nodes for the `MaxScoreIterator` to choose from. This keeps scheduling fast while producing acceptable results because the results are binpacked. Jobs with a `spread` block (or node affinity) remove this limit in order to produce correct spread scoring. This means that every allocation within a job with a `spread` block is evaluated against _all_ eligible nodes. Operators of large clusters have reported that jobs with `spread` blocks that are eligible on a large number of nodes can take longer than the nack timeout to evaluate (60s). Typical evaluations are processed in milliseconds. In practice, it's not necessary to evaluate every eligible node for every allocation on large clusters, because the `RandomIterator` at the base of the scheduler stack produces enough variation in each pass that the likelihood of an uneven spread is negligible. Note that feasibility is checked before the limit, so this only impacts the number of _eligible_ nodes available for scoring, not the total number of nodes. This changeset sets the iterator limit for "large" `spread` block and node affinity jobs to be equal to the number of desired allocations. This brings an example problematic job evaluation down from ~3min to ~10s. The included tests ensure that we have acceptable spread results across a variety of large cluster topologies. --- .changelog/11712.txt | 3 + scheduler/spread_test.go | 243 ++++++++++++++++++ scheduler/stack.go | 9 +- .../content/docs/job-specification/spread.mdx | 8 +- 4 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 .changelog/11712.txt diff --git a/.changelog/11712.txt b/.changelog/11712.txt new file mode 100644 index 000000000000..376da7c229ec --- /dev/null +++ b/.changelog/11712.txt @@ -0,0 +1,3 @@ +```release-note:bug +scheduler: Fixed a performance bug where `spread` and node affinity can cause a job to take longer than the nack timeout to be evaluated. +``` diff --git a/scheduler/spread_test.go b/scheduler/spread_test.go index c21a58f6b990..04040acb6755 100644 --- a/scheduler/spread_test.go +++ b/scheduler/spread_test.go @@ -2,7 +2,10 @@ package scheduler import ( "math" + "math/rand" + "sort" "testing" + "time" "fmt" @@ -568,3 +571,243 @@ func Test_evenSpreadScoreBoost(t *testing.T) { require.False(t, math.IsInf(boost, 1)) require.Equal(t, 1.0, boost) } + +// TestSpreadOnLargeCluster exercises potentially quadratic +// performance cases with spread scheduling when we have a large +// number of eligible nodes unless we limit the number that each +// MaxScore attempt considers. By reducing the total from MaxInt, we +// can prevent quadratic performance but then we need this test to +// verify we have satisfactory spread results. +func TestSpreadOnLargeCluster(t *testing.T) { + t.Parallel() + cases := []struct { + name string + nodeCount int + racks map[string]int + allocs int + }{ + { + name: "nodes=10k even racks=100 allocs=500", + nodeCount: 10000, + racks: generateEvenRacks(10000, 100), + allocs: 500, + }, + { + name: "nodes=10k even racks=100 allocs=50", + nodeCount: 10000, + racks: generateEvenRacks(10000, 100), + allocs: 50, + }, + { + name: "nodes=10k even racks=10 allocs=500", + nodeCount: 10000, + racks: generateEvenRacks(10000, 10), + allocs: 500, + }, + { + name: "nodes=10k even racks=10 allocs=50", + nodeCount: 10000, + racks: generateEvenRacks(10000, 10), + allocs: 500, + }, + { + name: "nodes=10k small uneven racks allocs=500", + nodeCount: 10000, + racks: generateUnevenRacks(t, 10000, 50), + allocs: 500, + }, + { + name: "nodes=10k small uneven racks allocs=50", + nodeCount: 10000, + racks: generateUnevenRacks(t, 10000, 50), + allocs: 500, + }, + { + name: "nodes=10k many uneven racks allocs=500", + nodeCount: 10000, + racks: generateUnevenRacks(t, 10000, 500), + allocs: 500, + }, + { + name: "nodes=10k many uneven racks allocs=50", + nodeCount: 10000, + racks: generateUnevenRacks(t, 10000, 500), + allocs: 50, + }, + } + + for i := range cases { + tc := cases[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + h := NewHarness(t) + err := upsertNodes(h, tc.nodeCount, tc.racks) + require.NoError(t, err) + job := generateJob(tc.allocs) + eval, err := upsertJob(h, job) + require.NoError(t, err) + + start := time.Now() + err = h.Process(NewServiceScheduler, eval) + require.NoError(t, err) + require.LessOrEqual(t, time.Since(start), time.Duration(60*time.Second), + "time to evaluate exceeded EvalNackTimeout") + + require.Len(t, h.Plans, 1) + require.False(t, h.Plans[0].IsNoOp()) + require.NoError(t, validateEqualSpread(h)) + }) + } +} + +// generateUnevenRacks creates a map of rack names to a count of nodes +// evenly distributed in those racks +func generateEvenRacks(nodes int, rackCount int) map[string]int { + racks := map[string]int{} + for i := 0; i < nodes; i++ { + racks[fmt.Sprintf("r%d", i%rackCount)]++ + } + return racks +} + +// generateUnevenRacks creates a random map of rack names to a count +// of nodes in that rack +func generateUnevenRacks(t *testing.T, nodes int, rackCount int) map[string]int { + rackNames := []string{} + for i := 0; i < rackCount; i++ { + rackNames = append(rackNames, fmt.Sprintf("r%d", i)) + } + + // print this so that any future test flakes can be more easily + // reproduced + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("nodes=%d racks=%d seed=%d\n", nodes, rackCount, seed) + + racks := map[string]int{} + for i := 0; i < nodes; i++ { + idx := rand.Intn(len(rackNames)) + racks[rackNames[idx]]++ + } + return racks +} + +// upsertNodes creates a collection of Nodes in the state store, +// distributed among the racks +func upsertNodes(h *Harness, count int, racks map[string]int) error { + + datacenters := []string{"dc-1", "dc-2"} + rackAssignments := []string{} + for rack, count := range racks { + for i := 0; i < count; i++ { + rackAssignments = append(rackAssignments, rack) + } + } + + for i := 0; i < count; i++ { + node := mock.Node() + node.Datacenter = datacenters[i%2] + node.Meta = map[string]string{} + node.Meta["rack"] = fmt.Sprintf("r%s", rackAssignments[i]) + node.NodeResources.Cpu.CpuShares = 14000 + node.NodeResources.Memory.MemoryMB = 32000 + err := h.State.UpsertNode(structs.MsgTypeTestSetup, h.NextIndex(), node) + if err != nil { + return err + } + } + return nil +} + +func generateJob(jobSize int) *structs.Job { + job := mock.Job() + job.Datacenters = []string{"dc-1", "dc-2"} + job.Spreads = []*structs.Spread{{Attribute: "${meta.rack}"}} + job.Constraints = []*structs.Constraint{} + job.TaskGroups[0].Count = jobSize + job.TaskGroups[0].Networks = nil + job.TaskGroups[0].Services = []*structs.Service{} + job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 6000, + MemoryMB: 6000, + } + return job +} + +func upsertJob(h *Harness, job *structs.Job) (*structs.Evaluation, error) { + err := h.State.UpsertJob(structs.MsgTypeTestSetup, h.NextIndex(), job) + if err != nil { + return nil, err + } + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + err = h.State.UpsertEvals(structs.MsgTypeTestSetup, + h.NextIndex(), []*structs.Evaluation{eval}) + if err != nil { + return nil, err + } + return eval, nil +} + +// validateEqualSpread compares the resulting plan to the node +// metadata to verify that each group of spread targets has an equal +// distribution. +func validateEqualSpread(h *Harness) error { + + iter, err := h.State.Nodes(nil) + if err != nil { + return err + } + i := 0 + nodesToRacks := map[string]string{} + racksToAllocCount := map[string]int{} + for { + raw := iter.Next() + if raw == nil { + break + } + node := raw.(*structs.Node) + rack, ok := node.Meta["rack"] + if ok { + nodesToRacks[node.ID] = rack + racksToAllocCount[rack] = 0 + } + i++ + } + + // Collapse the count of allocations per node into a list of + // counts. The results should be clustered within one of each + // other. + for nodeID, nodeAllocs := range h.Plans[0].NodeAllocation { + racksToAllocCount[nodesToRacks[nodeID]] += len(nodeAllocs) + } + countSet := map[int]int{} + for _, count := range racksToAllocCount { + countSet[count]++ + } + + countSlice := []int{} + for count := range countSet { + countSlice = append(countSlice, count) + } + + switch len(countSlice) { + case 1: + return nil + case 2, 3: + sort.Ints(countSlice) + for i := 1; i < len(countSlice); i++ { + if countSlice[i] != countSlice[i-1]+1 { + return fmt.Errorf("expected even distributon of allocs to racks, but got:\n%+v", countSet) + } + } + return nil + } + return fmt.Errorf("expected even distributon of allocs to racks, but got:\n%+v", countSet) +} diff --git a/scheduler/stack.go b/scheduler/stack.go index c9c8e3609319..d2b546107f73 100644 --- a/scheduler/stack.go +++ b/scheduler/stack.go @@ -163,7 +163,14 @@ func (s *GenericStack) Select(tg *structs.TaskGroup, options *SelectOptions) *Ra s.spread.SetTaskGroup(tg) if s.nodeAffinity.hasAffinities() || s.spread.hasSpreads() { - s.limit.SetLimit(math.MaxInt32) + // scoring spread across all nodes has quadratic behavior, so + // we need to consider a subset of nodes to keep evaluaton times + // reasonable but enough to ensure spread is correct. this + // value was empirically determined. + s.limit.SetLimit(tg.Count) + if tg.Count < 100 { + s.limit.SetLimit(100) + } } if contextual, ok := s.quota.(ContextualIterator); ok { diff --git a/website/content/docs/job-specification/spread.mdx b/website/content/docs/job-specification/spread.mdx index 0fab0f58ec09..3889e5623774 100644 --- a/website/content/docs/job-specification/spread.mdx +++ b/website/content/docs/job-specification/spread.mdx @@ -54,8 +54,12 @@ spread stanza. Spread scores are combined with other scoring factors such as bin A job or task group can have more than one spread criteria, with weights to express relative preference. -Spread criteria are treated as a soft preference by the Nomad scheduler. -If no nodes match a given spread criteria, placement is still successful. +Spread criteria are treated as a soft preference by the Nomad +scheduler. If no nodes match a given spread criteria, placement is +still successful. To avoid scoring every node for every placement, +allocations may not be perfectly spread. Spread works best on +attributes with similar number of nodes: identically configured racks +or similarly configured datacenters. Spread may be expressed on [attributes][interpolation] or [client metadata][client-meta]. Additionally, spread may be specified at the [job][job] and [group][group] levels for ultimate flexibility. Job level spread criteria are inherited by all task groups in the job.