add noiseDelta to telemetry test

agent/functional_tests/tests/functionaltests_test.go

@@ -56,6 +56,8 @@ const (
 	bytePerMegabyte          = 1024 * 1024
 	minimumCPUShares         = 128
 	maximumCPUShares         = 10240
+
+	statsNoiseDelta = 5.0
 )
 
 // TestPullInvalidImage verifies that an invalid image returns an error
@@ -511,11 +513,12 @@ func telemetryTest(t *testing.T, taskDefinition string) {
 	time.Sleep(waitMetricsInCloudwatchDuration)
 
 	cwclient := cloudwatch.New(session.New(), aws.NewConfig().WithRegion(*ECS.Config.Region))
-	_, err = VerifyMetrics(cwclient, params, true)
+
+	_, err = VerifyMetrics(cwclient, params, true, statsNoiseDelta)
 	assert.NoError(t, err, "Before task running, verify metrics for CPU utilization failed")
 
 	params.MetricName = aws.String("MemoryUtilization")
-	_, err = VerifyMetrics(cwclient, params, true)
+	_, err = VerifyMetrics(cwclient, params, true, statsNoiseDelta)
 	assert.NoError(t, err, "Before task running, verify metrics for memory utilization failed")
 
 	tdOverrides := make(map[string]string)
@@ -531,19 +534,21 @@ func telemetryTest(t *testing.T, taskDefinition string) {
 	params.EndTime = aws.Time(RoundTimeUp(time.Now(), time.Minute).UTC())
 	params.StartTime = aws.Time((*params.EndTime).Add(-waitMetricsInCloudwatchDuration).UTC())
 	params.MetricName = aws.String("CPUUtilization")
-	metrics, err := VerifyMetrics(cwclient, params, false)
+	metrics, err := VerifyMetrics(cwclient, params, false, 0.0)
 	assert.NoError(t, err, "Task is running, verify metrics for CPU utilization failed")
-	// Also verify the cpu usage is around expectedCPUPercentage +/- 5%
-	assert.InDelta(t, expectedCPUPercentage*100.0, *metrics.Average, 5)
+	// Also verify the cpu usage is around expectedCPUPercentage
+	// +/- StatsNoiseDelta percentage
+	assert.InDelta(t, expectedCPUPercentage*100.0, *metrics.Average, statsNoiseDelta)
 
 	params.MetricName = aws.String("MemoryUtilization")
-	metrics, err = VerifyMetrics(cwclient, params, false)
+	metrics, err = VerifyMetrics(cwclient, params, false, 0.0)
 	assert.NoError(t, err, "Task is running, verify metrics for memory utilization failed")
 	memInfo, err := system.ReadMemInfo()
 	require.NoError(t, err, "Acquiring system info failed")
 	totalMemory := memInfo.MemTotal / bytePerMegabyte
-	// Verify the memory usage is around 1024/totalMemory +/- 5%
-	assert.InDelta(t, float32(1024*100)/float32(totalMemory), *metrics.Average, 5)
+	// Verify the memory usage is around 1024/totalMemory
+	// +/- StatsNoiseDelta percentage
+	assert.InDelta(t, float32(1024*100)/float32(totalMemory), *metrics.Average, statsNoiseDelta)
 
 	err = testTask.Stop()
 	require.NoError(t, err, "Failed to stop the telemetry task")
@@ -555,11 +560,11 @@ func telemetryTest(t *testing.T, taskDefinition string) {
 	params.EndTime = aws.Time(RoundTimeUp(time.Now(), time.Minute).UTC())
 	params.StartTime = aws.Time((*params.EndTime).Add(-waitMetricsInCloudwatchDuration).UTC())
 	params.MetricName = aws.String("CPUUtilization")
-	_, err = VerifyMetrics(cwclient, params, true)
+	_, err = VerifyMetrics(cwclient, params, true, statsNoiseDelta)
 	assert.NoError(t, err, "Task stopped: verify metrics for CPU utilization failed")
 
 	params.MetricName = aws.String("MemoryUtilization")
-	_, err = VerifyMetrics(cwclient, params, true)
+	_, err = VerifyMetrics(cwclient, params, true, statsNoiseDelta)
 	assert.NoError(t, err, "Task stopped, verify metrics for memory utilization failed")
 }
 
@@ -621,13 +626,13 @@ func telemetryTestWithStatsPolling(t *testing.T, taskDefinition string) {
 	params.EndTime = aws.Time(RoundTimeUp(time.Now(), time.Minute).UTC())
 	params.StartTime = aws.Time((*params.EndTime).Add(-waitMetricsInCloudwatchDuration).UTC())
 	params.MetricName = aws.String("CPUUtilization")
-	metrics, err := VerifyMetrics(cwclient, params, false)
+	metrics, err := VerifyMetrics(cwclient, params, false, 0.0)
 	assert.NoError(t, err, "Task is running, verify metrics for CPU utilization failed")
 	// Also verify the cpu usage is around expectedCPUPercentage +/- 5%
-	assert.InDelta(t, expectedCPUPercentage*100.0, *metrics.Average, 5)
+	assert.InDelta(t, expectedCPUPercentage*100.0, *metrics.Average, statsNoiseDelta)
 
 	params.MetricName = aws.String("MemoryUtilization")
-	metrics, err = VerifyMetrics(cwclient, params, false)
+	metrics, err = VerifyMetrics(cwclient, params, false, 0.0)
 	assert.NoError(t, err, "Task is running, verify metrics for memory utilization failed")
 	memInfo, err := system.ReadMemInfo()
 	require.NoError(t, err, "Acquiring system info failed")

agent/functional_tests/util/utils.go

@@ -334,7 +334,9 @@ func DeleteCluster(t *testing.T, clusterName string) {
 
 // VerifyMetrics whether the response is as expected
 // the expected value can be 0 or positive
-func VerifyMetrics(cwclient *cloudwatch.CloudWatch, params *cloudwatch.GetMetricStatisticsInput, idleCluster bool) (*cloudwatch.Datapoint, error) {
+// noiseDelta should be significantly less than the percentage of cpu/memory we
+// use for non-idle workload.
+func VerifyMetrics(cwclient *cloudwatch.CloudWatch, params *cloudwatch.GetMetricStatisticsInput, idleCluster bool, noiseDelta float64) (*cloudwatch.Datapoint, error) {
 	resp, err := cwclient.GetMetricStatistics(params)
 	if err != nil {
 		return nil, fmt.Errorf("Error getting metrics of cluster: %v", err)
@@ -353,14 +355,13 @@ func VerifyMetrics(cwclient *cloudwatch.CloudWatch, params *cloudwatch.GetMetric
 	if *datapoint.SampleCount != 1.0 {
 		return nil, fmt.Errorf("Incorrect SampleCount %f, expected 1", *datapoint.SampleCount)
 	}
-
 	if idleCluster {
-		if *datapoint.Average != 0.0 {
-			return nil, fmt.Errorf("non-zero utilization for idle cluster")
+		if *datapoint.Average >= noiseDelta {
+			return nil, fmt.Errorf("utilization is >= expected noise delta for idle cluster")
 		}
 	} else {
-		if *datapoint.Average == 0.0 {
-			return nil, fmt.Errorf("utilization is zero for non-idle cluster")
+		if *datapoint.Average < noiseDelta {
+			return nil, fmt.Errorf("utilization is < expected noise delta for non-idle cluster")
 		}
 	}
 	return datapoint, nil