Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Bug] Fix randomly failing tests #1916

Merged
merged 1 commit into from
Oct 7, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion AISKU/Tests/Unit/src/ApplicationInsightsDeprecatedTests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { Snippet } from "../../../src/Initialization";
import { Sender } from "@microsoft/applicationinsights-channel-js";
import { SinonSpy } from "sinon";
import { AITestClass, Assert, PollingAssert } from "@microsoft/ai-test-framework";
import { dumpObj } from "@microsoft/applicationinsights-core-js";

export class ApplicationInsightsDeprecatedTests extends AITestClass {
private static readonly _instrumentationKey = 'b7170927-2d1c-44f1-acec-59f4e1751c11';
Expand Down Expand Up @@ -142,7 +143,7 @@ export class ApplicationInsightsDeprecatedTests extends AITestClass {
Assert.ok(isValidCallCount, "logging spy was called 0 time(s)");
if (!isValidCallCount) {
while (this.loggingSpy.args.length) {
Assert.ok(false, "[warning thrown]: " + this.loggingSpy.args.pop());
Assert.ok(false, "[warning thrown]: " + dumpObj(this.loggingSpy.args.pop()));
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions AISKU/Tests/Unit/src/SnippetInitialization.Tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Sender } from "@microsoft/applicationinsights-channel-js";
import { SinonSpy } from "sinon";
import { AITestClass, Assert, PollingAssert } from "@microsoft/ai-test-framework";
import { createSnippetV5 } from "./testSnippet";
import { hasOwnProperty, isNotNullOrUndefined, ITelemetryItem, objForEachKey } from "@microsoft/applicationinsights-core-js";
import { dumpObj, hasOwnProperty, isNotNullOrUndefined, ITelemetryItem, objForEachKey } from "@microsoft/applicationinsights-core-js";
import { ContextTagKeys, DistributedTracingModes, IConfig, IDependencyTelemetry, RequestHeaders, Util } from "@microsoft/applicationinsights-common";
import { getGlobal } from "@microsoft/applicationinsights-shims";
import { TelemetryContext } from "@microsoft/applicationinsights-properties-js";
Expand Down Expand Up @@ -904,7 +904,7 @@ export class SnippetInitializationTests extends AITestClass {
Assert.ok(isValidCallCount, "logging spy was called 0 time(s)");
if (!isValidCallCount) {
while (this.loggingSpy.args.length) {
Assert.ok(false, "[warning thrown]: " + this.loggingSpy.args.pop());
Assert.ok(false, "[warning thrown]: " + dumpObj(this.loggingSpy.args.pop()));
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions AISKU/Tests/Unit/src/SnippetLegacyInitialization.Tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Sender } from "@microsoft/applicationinsights-channel-js";
import { createLegacySnippet } from "./testLegacySnippet";
import { SinonSpy } from "sinon";
import { AITestClass, Assert, PollingAssert } from "@microsoft/ai-test-framework";
import { hasOwnProperty, isNotNullOrUndefined } from "@microsoft/applicationinsights-core-js";
import { dumpObj, hasOwnProperty, isNotNullOrUndefined } from "@microsoft/applicationinsights-core-js";

function getBasicLegacySnippetConfig() {
return {
Expand Down Expand Up @@ -342,7 +342,7 @@ export class SnippetLegacyInitializationTests extends AITestClass {
Assert.ok(isValidCallCount, "logging spy was called 0 time(s)");
if (!isValidCallCount) {
while (this.loggingSpy.args.length) {
Assert.ok(false, "[warning thrown]: " + this.loggingSpy.args.pop());
Assert.ok(false, "[warning thrown]: " + dumpObj(this.loggingSpy.args.pop()));
}
}
}
Expand Down
12 changes: 11 additions & 1 deletion AISKU/Tests/Unit/src/applicationinsights.e2e.fetch.tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export class ApplicationInsightsFetchTests extends ApplicationInsightsTests {
disableXhr: true, // Disable XHR support
enableRequestHeaderTracking: true,
enableResponseHeaderTracking: true,
maxBatchInterval: 2500,
maxBatchInterval: 500,
disableExceptionTracking: false,
namePrefix: sessionPrefix,
enableCorsCorrelation: true,
Expand All @@ -27,4 +27,14 @@ export class ApplicationInsightsFetchTests extends ApplicationInsightsTests {
convertUndefined: "test-value"
};
}

public testInitialize() {
super.testInitialize();

// Use the fake server for fetch tests as multiple test runs are causing timeout issues
// this.useFakeServer = true;
// this.fakeServerAutoRespond = true;
this.useFakeFetch = true;
this.fakeFetchAutoRespond = true;
}
}
5 changes: 2 additions & 3 deletions AISKU/Tests/Unit/src/applicationinsights.e2e.tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { SinonSpy } from 'sinon';
import { ApplicationInsights, IApplicationInsights } from '../../../src/applicationinsights-web'
import { Sender } from '@microsoft/applicationinsights-channel-js';
import { IDependencyTelemetry, ContextTagKeys, Util, Event, Trace, Exception, Metric, PageView, PageViewPerformance, RemoteDependencyData, DistributedTracingModes, RequestHeaders, IAutoExceptionTelemetry } from '@microsoft/applicationinsights-common';
import { AppInsightsCore, ITelemetryItem, getGlobal } from "@microsoft/applicationinsights-core-js";
import { AppInsightsCore, ITelemetryItem, getGlobal, dumpObj } from "@microsoft/applicationinsights-core-js";
import { TelemetryContext } from '@microsoft/applicationinsights-properties-js';


Expand Down Expand Up @@ -80,7 +80,6 @@ export class ApplicationInsightsTests extends AITestClass {
Assert.equal("4.0", item.ver, "Telemetry items inside telemetry initializers should be in CS4.0 format");
});


// Setup Sinon stuff
const sender: Sender = this._ai.appInsights.core.getTransmissionControls()[0][0] as Sender;
this.errorSpy = this.sandbox.spy(sender, '_onError');
Expand Down Expand Up @@ -973,7 +972,7 @@ export class ApplicationInsightsTests extends AITestClass {
Assert.ok(isValidCallCount, "logging spy was called 0 time(s)");
if (!isValidCallCount) {
while (this.loggingSpy.args.length) {
Assert.ok(false, "[warning thrown]: " + this.loggingSpy.args.pop());
Assert.ok(false, "[warning thrown]: " + dumpObj(this.loggingSpy.args.pop()));
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions AISKU/Tests/Unit/src/sender.e2e.tests.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { ApplicationInsights, IApplicationInsights } from '../../../src/applicationinsights-web'
import { Sender } from '@microsoft/applicationinsights-channel-js';
import { Util } from '@microsoft/applicationinsights-common';
import { getJSON } from '@microsoft/applicationinsights-core-js';
import { dumpObj, getJSON } from '@microsoft/applicationinsights-core-js';
import { SinonSpy } from 'sinon';
import { Assert, AITestClass, PollingAssert} from "@microsoft/ai-test-framework"

Expand Down Expand Up @@ -174,7 +174,7 @@ export class SenderE2ETests extends AITestClass {
Assert.ok(isValidCallCount, "logging spy was called 0 time(s)");
if (!isValidCallCount) {
while (this.loggingSpy.args.length) {
Assert.ok(false, "[warning thrown]: " + this.loggingSpy.args.pop());
Assert.ok(false, "[warning thrown]: " + dumpObj(this.loggingSpy.args.pop()));
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion AISKU/Tests/Unit/src/validate.e2e.tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ApplicationInsights, IApplicationInsights } from '../../../src/applicat
import { Sender } from '@microsoft/applicationinsights-channel-js';
import { SinonSpy } from 'sinon';
import { AITestClass, Assert, PollingAssert } from '@microsoft/ai-test-framework';
import { dumpObj } from '@microsoft/applicationinsights-core-js';

export class ValidateE2ETests extends AITestClass {
private readonly _instrumentationKey = 'b7170927-2d1c-44f1-acec-59f4e1751c11';
Expand Down Expand Up @@ -153,7 +154,7 @@ export class ValidateE2ETests extends AITestClass {
Assert.ok(isValidCallCount, "logging spy was called 0 time(s)");
if (!isValidCallCount) {
while (this.loggingSpy.args.length) {
Assert.ok(false, "[warning thrown]: " + this.loggingSpy.args.pop());
Assert.ok(false, "[warning thrown]: " + dumpObj(this.loggingSpy.args.pop()));
}
}
}
Expand Down
90 changes: 84 additions & 6 deletions common/Tests/Framework/src/AITestClass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ export interface FakeXMLHttpRequest extends XMLHttpRequest {
respond: (status: number, headers: any, body: string) => void;
}

export interface IFetchArgs {
input: RequestInfo,
init: RequestInit
}

function _getObjName(target:any, unknownValue?:string) {
if (target.hasOwnProperty("prototype")) {
// Look like a prototype
Expand Down Expand Up @@ -76,6 +81,7 @@ export class AITestClass {
protected clock: any;
public sandbox: SinonSandbox;
public fakeServerAutoRespond: boolean = false;
public fakeFetchAutoRespond: boolean = false;
public isEmulatingEs3: boolean;

/**
Expand All @@ -93,10 +99,12 @@ export class AITestClass {

/** Turns on/off sinon's fake implementation of XMLHttpRequest. On by default. */
private _useFakeServer: boolean = true;
private _useFakeFetch: boolean = false;
private _moduleName: string;
private _xhr: any;
private _xhrOrgSend: any;
private _xhrRequests: FakeXMLHttpRequest[] = [];
private _fetchRequests: IFetchArgs[] = [];
private _orgNavigator: any;
private _orgPerformance: any;
private _beaconHooks: any[] = [];
Expand Down Expand Up @@ -130,6 +138,19 @@ export class AITestClass {
}
}

get useFakeFetch(): boolean {
return this._useFakeFetch;
}

set useFakeFetch(value: boolean) {
this._useFakeFetch = value;
if (!value) {
this._unhookFetch();
} else if (value && AITestClass.currentTestInfo) {
this._hookFetch();
}
}

/** Method called before the start of each test method */
public testInitialize() {
}
Expand Down Expand Up @@ -217,6 +238,15 @@ export class AITestClass {
self._hookXhr();
}

let useFakeFetch = testInfo.useFakeFetch;
if (useFakeFetch === undefined) {
useFakeFetch = self.useFakeFetch;
}

if (useFakeFetch && !self.isEmulatingEs3) {
self._hookFetch();
}

if (testInfo.useFakeTimers) {
self.clock = sinon.useFakeTimers();
}
Expand Down Expand Up @@ -910,13 +940,13 @@ export class AITestClass {
/** Called when the test is completed. */
private _testCompleted(failed?: boolean) {
this._unhookXhr();
this._unhookFetch();

if (this.clock) {
this.clock.restore();
this.clock = null;
}


if (this._orgCrypto && window.crypto !== this._orgCrypto) {
this.setCrypto(this._orgCrypto);
}
Expand Down Expand Up @@ -1038,11 +1068,7 @@ export class AITestClass {
this._restoreObject(this._orgObjectFuncs);
this._orgObjectFuncs = null;

if (this._orgFetch) {
let global = window as any;
global.fetch = this._orgFetch;
this._orgFetch = null;
}
this._unhookFetch();
}

private _emulateEs3() {
Expand Down Expand Up @@ -1112,6 +1138,58 @@ export class AITestClass {
}
}

private _hookFetch() {
let _self = this;
if (!this._orgFetch) {
let global = window as any;
this._orgFetch = global.fetch;
_self._fetchRequests = [];
global.fetch = function(input: RequestInfo, init?: RequestInit) {
let theFetch = this;
_self._fetchRequests.push({
input,
init
});

let theArguments = arguments;
let autoRespond = _self.fakeFetchAutoRespond;
if (AITestClass.currentTestInfo && AITestClass.currentTestInfo.fakeFetchAutoRespond !== undefined) {
autoRespond = AITestClass.currentTestInfo.fakeFetchAutoRespond;
}

if (autoRespond) {
return new Promise((resolve, reject) => {
AITestClass.orgSetTimeout && AITestClass.orgSetTimeout(() => {
if (AITestClass.currentTestInfo) {
let theResponse = {
itemsReceived: 1,
itemsAccepted: 1,
errors: [] as any[],
appId: "00000000-0000-0000-0000-000000000000"
};
resolve(new Response(JSON.stringify(theResponse), {
headers: [],
status: 200,
statusText: "Ok"
}));
}
}, 5);
});
}

return _self._orgFetch.apply(theFetch, theArguments);
};
}
}

private _unhookFetch() {
if (this._orgFetch) {
let global = window as any;
global.fetch = this._orgFetch;
this._orgFetch = null;
}
}

private _assertNoEvents(target: any, targetName: string): void {
let failed = false;
_getAllAiDataKeys(target, (name, value) => {
Expand Down
6 changes: 6 additions & 0 deletions common/Tests/Framework/src/TestCase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ export class TestCase {
useFakeServer?: boolean;
fakeServerAutoRespond?: boolean;

useFakeFetch?: boolean;
fakeFetchAutoRespond?: boolean;

useFakeTimers?: boolean;

/** Test case method */
Expand Down Expand Up @@ -55,6 +58,9 @@ export interface TestCaseAsync {
useFakeServer?: boolean;
fakeServerAutoRespond?: boolean;

useFakeFetch?: boolean;
fakeFetchAutoRespond?: boolean;

useFakeTimers?: boolean;

/** time to wait after pre before invoking post and calling start() */
Expand Down
12 changes: 6 additions & 6 deletions common/config/rush/npm-shrinkwrap.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Assert, AITestClass } from "@microsoft/ai-test-framework";
import { AppInsightsCore, IConfiguration, DiagnosticLogger, ITelemetryItem, createCookieMgr, newId, strTrim } from "@microsoft/applicationinsights-core-js";
import { AppInsightsCore, IConfiguration, DiagnosticLogger, ITelemetryItem, createCookieMgr, newId, strTrim, random32 } from "@microsoft/applicationinsights-core-js";
import PropertiesPlugin from "../../../src/PropertiesPlugin";
import { ITelemetryConfig } from "../../../src/Interfaces/ITelemetryConfig";
import { TelemetryContext } from "../../../src/TelemetryContext";
Expand Down Expand Up @@ -167,11 +167,15 @@ export class PropertiesTests extends AITestClass {

this.testCase({
name: "ai_user cookie is set with acq date and year expiration",
useFakeTimers: true,
test: () => {
// setup
var actualCookieName: string;
var actualCookieValue: string;

// Just move the time forward a random amount of time so that the cookie time is different for different test runs
this.clock.tick(random32());

var newIdStub = this.sandbox.stub(this as any, "_getNewId").callsFake(() => "newId");
var getCookieStub = this.sandbox.stub(this as any, "_getCookie").callsFake(() =>"");
var setCookieStub = this.sandbox.stub(this as any, "_setCookie").callsFake((cookieName, cookieValue) => {
Expand Down