UI: Retry stats requests

ui/app/components/allocation-row.js

@@ -1,6 +1,7 @@
 import Ember from 'ember';
 import { inject as service } from '@ember/service';
 import Component from '@ember/component';
+import { computed } from '@ember/object';
 import { run } from '@ember/runloop';
 import { lazyClick } from '../helpers/lazy-click';
 import { task, timeout } from 'ember-concurrency';
@@ -17,10 +18,14 @@ export default Component.extend({
   // Used to determine whether the row should mention the node or the job
   context: null,
 
+  backoffSequence: computed(() => [500, 800, 1300, 2100, 3400, 5500]),
+
   // Internal state
   stats: null,
   statsError: false,
 
+  enablePolling: computed(() => !Ember.testing),
+
   onClick() {},
 
   click(event) {
@@ -63,19 +68,19 @@ export default Component.extend({
   },
 
   fetchStats: task(function*(allocation) {
-    const maxTiming = 5500;
-    const backoffSequence = [500, 800, 1300, 2100, 3400];
+    const backoffSequence = this.get('backoffSequence').slice();
+    const maxTiming = backoffSequence.pop();
 
     do {
       try {
         const stats = yield allocation.fetchStats();
         this.set('stats', stats);
+        this.set('statsError', false);
       } catch (error) {
         this.set('statsError', true);
-        break;
       }
       yield timeout(backoffSequence.shift() || maxTiming);
-    } while (!Ember.testing);
+    } while (this.get('enablePolling'));
   }).drop(),
 });
 

ui/mirage/config.js

@@ -132,6 +132,8 @@ export default function() {
     return this.serialize(allocations.where({ nodeId: params.id }));
   });
 
+  this.get('/allocations');
+
   this.get('/allocation/:id');
 
   this.get('/namespaces', function({ namespaces }) {

ui/mirage/data/generate-resources.js

@@ -0,0 +1,22 @@
+export default function generateResources() {
+  return {
+    CpuStats: {
+      Measured: ['Throttled Periods', 'Throttled Time', 'Percent'],
+      Percent: 0.14159538847117795,
+      SystemMode: 0,
+      ThrottledPeriods: 0,
+      ThrottledTime: 0,
+      TotalTicks: 300.256693934837093,
+      UserMode: 0,
+    },
+    MemoryStats: {
+      Cache: 1744896,
+      KernelMaxUsage: 0,
+      KernelUsage: 0,
+      MaxUsage: 4710400,
+      Measured: ['RSS', 'Cache', 'Swap', 'Max Usage'],
+      RSS: 1486848009,
+      Swap: 0,
+    },
+  };
+}

ui/mirage/factories/client-allocation-stats.js

@@ -1,4 +1,5 @@
 import { Factory } from 'ember-cli-mirage';
+import generateResources from '../data/generate-resources';
 
 export default Factory.extend({
   resourceUsage: generateResources,
@@ -17,26 +18,3 @@ export default Factory.extend({
     return hash;
   },
 });
-
-function generateResources() {
-  return {
-    CpuStats: {
-      Measured: ['Throttled Periods', 'Throttled Time', 'Percent'],
-      Percent: 0.14159538847117795,
-      SystemMode: 0,
-      ThrottledPeriods: 0,
-      ThrottledTime: 0,
-      TotalTicks: 3.256693934837093,
-      UserMode: 0,
-    },
-    MemoryStats: {
-      Cache: 1744896,
-      KernelMaxUsage: 0,
-      KernelUsage: 0,
-      MaxUsage: 4710400,
-      Measured: ['RSS', 'Cache', 'Swap', 'Max Usage'],
-      RSS: 1486848,
-      Swap: 0,
-    },
-  };
-}

ui/tests/integration/allocation-row-test.js

@@ -0,0 +1,85 @@
+import { getOwner } from '@ember/application';
+import { test, moduleForComponent } from 'ember-qunit';
+import wait from 'ember-test-helpers/wait';
+import hbs from 'htmlbars-inline-precompile';
+import generateResources from '../../mirage/data/generate-resources';
+import { startMirage } from 'nomad-ui/initializers/ember-cli-mirage';
+import Response from 'ember-cli-mirage/response';
+
+moduleForComponent('allocation-row', 'Integration | Component | allocation row', {
+  integration: true,
+  beforeEach() {
+    this.store = getOwner(this).lookup('service:store');
+    this.server = startMirage();
+    this.server.create('namespace');
+    this.server.create('node');
+    this.server.create('job', { createAllocations: false });
+  },
+  afterEach() {
+    this.server.shutdown();
+  },
+});
+
+test('Allocation row polls for stats, even when it errors or has an invalid response', function(assert) {
+  const component = this;
+
+  let currentFrame = 0;
+  let frames = [
+    JSON.stringify({ ResourceUsage: generateResources() }),
+    JSON.stringify({ ResourceUsage: generateResources() }),
+    null,
+    '<Not>Valid JSON</Not>',
+    JSON.stringify({ ResourceUsage: generateResources() }),
+  ];
+  const backoffSequence = [50];
+
+  this.server.get('/client/allocation/:id/stats', function() {
+    const response = frames[++currentFrame];
+
+    // Disable polling to stop the EC task in the component
+    if (currentFrame >= frames.length) {
+      component.set('enablePolling', false);
+    }
+
+    if (response) {
+      return response;
+    }
+    return new Response(500, {}, '');
+  });
+
+  this.server.create('allocation');
+  this.store.findAll('allocation');
+
+  let allocation;
+
+  return wait()
+    .then(() => {
+      allocation = this.store.peekAll('allocation').get('firstObject');
+
+      this.setProperties({
+        allocation,
+        backoffSequence,
+        context: 'job',
+        enablePolling: true,
+      });
+
+      this.render(hbs`
+        {{allocation-row
+          allocation=allocation
+          context=context
+          backoffSequence=backoffSequence
+          enablePolling=enablePolling}}
+      `);
+      return wait();
+    })
+    .then(() => {
+      assert.equal(
+        this.server.pretender.handledRequests.filterBy(
+          'url',
+          `/v1/client/allocation/${allocation.get('id')}/stats`
+        ).length,
+        frames.length,
+        'Requests continue to be made after malformed responses and server errors'
+      );
+    });
+});