-
Notifications
You must be signed in to change notification settings - Fork 240
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Karlie/refactor ai SKU test and add PerfTest (#1707)
* add perftest * update AISKU perf tests * updates * remove key * Update AISku Perf Tests to use the standard deployed scripts (not the *.gbl cersions) * enable perfevent for versions without doperf * update npm-shrinkwrap.json to 2.7.0 * change default version to 2.7.1 * remove extra key Co-authored-by: Nev Wylie <54870357+MSNev@users.noreply.github.com>
- Loading branch information
1 parent
91ee603
commit 6ac63f8
Showing
38 changed files
with
622 additions
and
1,684 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,361 @@ | ||
import { AITestClass, Assert } from "@microsoft/ai-test-framework"; | ||
import {AppInsightsInitPerfTestClass} from "./AISKUPerf"; | ||
|
||
function isNullOrUndefined(value: any): boolean { | ||
return value === undefined || value === null; | ||
} | ||
|
||
function createTrackEvent(eventName: string, type: string = "EventData", fieldNum?: number) { | ||
let number = fieldNum? fieldNum: Math.random() * 10 + 10; | ||
let fields = {}; | ||
for (let i=1; i <= number; i++) { | ||
let field = "field" + i; | ||
fields[field] = "value" + i; | ||
} | ||
return {name: eventName, properties: fields, baseType: type }; | ||
} | ||
|
||
function sendEventWithCollectorUrl(event: any) { | ||
let oneDS = window["oneDS"]; | ||
const generatedUrl = oneDS.getCollectorUrlGenerator(event); | ||
let collectorUrlScript = document.createElement("script"); | ||
collectorUrlScript.setAttribute("id", "collectorUrl"); | ||
collectorUrlScript.src=generatedUrl; | ||
document.head.appendChild(collectorUrlScript); | ||
collectorUrlScript.onload=collectorUrlScript["onreadystatechange"] = function() { | ||
Assert.ok(true,"telemetry sent"); | ||
} | ||
} | ||
|
||
function pushPerfEvents(event: any, AISKUPerfTest: any): void { | ||
if (!AISKUPerfTest.doFlush) { | ||
AISKUPerfTest.perfEventsBuffer.push(event); | ||
} | ||
else { | ||
AISKUPerfTest.perfEventWaitBuffer.push(event); | ||
} | ||
} | ||
|
||
function flush(AISKUPerfTest: any): void { | ||
AISKUPerfTest.doFlush = true; | ||
console.log("flush " + AISKUPerfTest.perfEventsBuffer.length +" events"); | ||
AISKUPerfTest.perfEventsBuffer.forEach((event) => { | ||
if (event.baseData) {sendEventWithCollectorUrl(event);} | ||
}) | ||
AISKUPerfTest.perfEventsBuffer = AISKUPerfTest.perfEventWaitBuffer.slice(0); | ||
AISKUPerfTest.doFlush = false; | ||
AISKUPerfTest.perfEventWaitBuffer = []; | ||
Assert.ok(true, "flush triggered"); | ||
} | ||
|
||
const TENANT_KEY = "2252db2e5e344635a36c5f1c04b3902c-eaced1c8-a046-4e8d-8fa8-c1ecf2077a5d-7256" | ||
function createPerfEvent(AISKUInitPerf: any, name: string, value: number, isProcessTime: boolean, msg?: string): void { | ||
if (isNullOrUndefined(value) || value < 0 || isNullOrUndefined(AISKUInitPerf)) return; | ||
let metricVal = isProcessTime? "ProcessTime":"UsedJSHeapSize"; | ||
let unit = isProcessTime? "ms":"KB"; | ||
let event = { | ||
name: "SDKPerfTest", | ||
iKey: TENANT_KEY, | ||
ver: "4.0", | ||
ext: {}, | ||
baseData: { | ||
testName: name, | ||
sku:AISKUInitPerf.skuName, | ||
version: AISKUInitPerf.version, | ||
unitOfMeasure: unit, | ||
metric: metricVal, | ||
value: value | ||
} | ||
}; | ||
|
||
pushPerfEvents(event, AISKUInitPerf); | ||
let message = msg? msg :`perfEvent: ${event.baseData.testName} ${event.baseData.value}${event.baseData.unitOfMeasure} added`; | ||
console.log(message); | ||
Assert.ok(true, message); | ||
} | ||
|
||
function parseBatchSendEvent(perfEvent: any, AISKUInitPerf: any, memoryUsageMarks: any): void { | ||
let curMemory = performance["memory"]?.usedJSHeapSize; | ||
let memoryMarks = Object.keys(memoryUsageMarks); | ||
var index = ""; | ||
|
||
if (memoryMarks && memoryMarks.length && curMemory !== undefined) { | ||
index = memoryMarks[memoryMarks.length-1]; | ||
createPerfEvent(AISKUInitPerf, perfEvent.name+index, perfEvent?.exTime, true); | ||
} | ||
} | ||
|
||
function parseTrackEvent(perfEvent: any, AISKUInitPerf: any): void { | ||
let payloadItem = perfEvent["payload"]?.item; | ||
let baseType = null; | ||
if (perfEvent["time"] && payloadItem) { | ||
baseType = payloadItem["baseType"]; | ||
} | ||
// skip PageviewPerformanceData && MessageData | ||
if (isNullOrUndefined(baseType) || baseType == "PageviewPerformanceData" || baseType == "MessageData") return; | ||
|
||
createPerfEvent(AISKUInitPerf, `Track${baseType}`, perfEvent?.time, true); | ||
var childEvts = perfEvent?.childEvts; | ||
while (childEvts?.length) { | ||
let curEvt = childEvts[0]; | ||
let name = `${baseType}-${curEvt.name}`; | ||
let value = curEvt?.exTime !== undefined ? curEvt.exTime:curEvt.time; | ||
createPerfEvent(AISKUInitPerf, name, value, true); | ||
childEvts = curEvt?.childEvts; | ||
} | ||
} | ||
|
||
function parseAppInsightsPerfEvent(perfEvent: any, AISKUInitPerf: any, memoryUsageMarks: any): void { | ||
let perfEventName = perfEvent?.name; | ||
switch (perfEventName) { | ||
case "AISKU.flush": | ||
parseBatchSendEvent(perfEvent, AISKUInitPerf, memoryUsageMarks); | ||
break; | ||
case "AppInsightsCore:track": | ||
parseTrackEvent(perfEvent, AISKUInitPerf); | ||
break; | ||
} | ||
} | ||
|
||
function addSnippetLoadingTimeEvent(AISKUPerfTest: any, endtime: number): void { | ||
let rawTime = endtime - AISKUPerfTest.snippetStartTime; | ||
var duration = Math.round(rawTime*1000)/1000; | ||
createPerfEvent(AISKUPerfTest, "SDKInit", duration, true, `AppInsightsInit-Init: Script LoadingTime: ${duration} ms added`); | ||
} | ||
|
||
|
||
export class AISKUPerf extends AITestClass { | ||
public AISKUPerfTest: AppInsightsInitPerfTestClass; | ||
public perfMgr: any; | ||
public doPerf: any; | ||
public appInsights: any; | ||
public initialMemoryUsage: number = 0; | ||
public batchStartTimeMarks: any; | ||
public memoryUsageMarks: any; | ||
public perfMgrSrc: any = null; | ||
|
||
public testInitialize() { | ||
try { | ||
this.AISKUPerfTest = new AppInsightsInitPerfTestClass(); | ||
this.AISKUPerfTest.loadScriptOnInit = () => {return this._loadScriptOnInit();} | ||
Assert.ok(window["oneDS"], "oneDS exists"); | ||
Assert.ok(window["Microsoft"]?.ApplicationInsights?.PerfMarkMeasureManager, "perfMgr exists"); | ||
Assert.ok(window["Microsoft"]?.ApplicationInsights?.doPerf, "doPerf exists"); | ||
this.perfMgr = window["Microsoft"].ApplicationInsights.PerfMarkMeasureManager; | ||
this.doPerf = window["Microsoft"].ApplicationInsights.doPerf; | ||
window["appInsightsInitPerftest"] = this.AISKUPerfTest; | ||
this.memoryUsageMarks = {}; | ||
this.batchStartTimeMarks = {}; | ||
|
||
} catch (e) { | ||
console.error("Failed to initialize AISKUPerf Tests", e); | ||
} | ||
} | ||
|
||
public testCleanup() { | ||
} | ||
|
||
public registerTests() { | ||
this.addPerfTest(); | ||
} | ||
|
||
constructor() { | ||
super("AISKUPerfTest"); | ||
} | ||
|
||
public addPerfTest(): void { | ||
this.testCaseAsync({ | ||
name: "AppInsights AISKU perf Test", | ||
stepDelay: 10000, | ||
steps: [() => { | ||
Assert.ok(window["appInsightsInitPerftest"], "global appInsightsInitPerftest exists"); | ||
Assert.ok(window["oneDS"], "oneDS exists"); | ||
Assert.ok(this.perfMgr, "perfMgr exists"); | ||
Assert.ok(this.doPerf, "doPerf exists"); | ||
console.log(this.AISKUPerfTest.version); | ||
|
||
try { | ||
if (!this.AISKUPerfTest.hasPerfMgr) { | ||
this.perfMgrSrc = new this.perfMgr({ | ||
useMarks: true, | ||
useEndMarks: true, | ||
uniqueNames: true | ||
}, | ||
{ | ||
perfEvent: (perfEvent) => { | ||
parseAppInsightsPerfEvent(perfEvent,this.AISKUPerfTest, this.memoryUsageMarks); | ||
} | ||
}); | ||
} | ||
this.initialMemoryUsage = performance["memory"]?.usedJSHeapSize; | ||
this._loadSnippet(); | ||
} catch (e) { | ||
Assert.ok(false, "load snippet error: " + e); | ||
} | ||
|
||
}].concat(() => { | ||
Assert.ok(true, "test version: " + this.AISKUPerfTest.version); | ||
}) | ||
}); | ||
} | ||
|
||
protected _loadScriptOnInit(): void { | ||
var snippetLoadingEndTime = performance.now(); | ||
this._addMemoryPerfEvent(this.initialMemoryUsage); | ||
Assert.ok(true,"AppInsights script loaded"); | ||
|
||
addSnippetLoadingTimeEvent(this.AISKUPerfTest,snippetLoadingEndTime); | ||
|
||
let appInsights = window["appInsights"]; | ||
this.appInsights = appInsights; | ||
|
||
try { | ||
let notificationManager = this.appInsights.core["_notificationManager"] ||this.appInsights.core?.getNotifyMgr(); | ||
if (notificationManager) { | ||
notificationManager.addNotificationListener({ | ||
perfEvent: (perfEvent: any) => { | ||
parseAppInsightsPerfEvent(perfEvent,this.AISKUPerfTest, this.memoryUsageMarks); | ||
}}); | ||
} | ||
|
||
} catch (e) { | ||
console.error(e); | ||
} | ||
|
||
this.addPerfEvents(); | ||
|
||
setTimeout(() => { | ||
flush(this.AISKUPerfTest); | ||
}, 9000); | ||
} | ||
|
||
protected _loadSnippet():void { | ||
this.initialMemoryUsage = performance["memory"]?.usedJSHeapSize; | ||
var tag = document.createElement("script"); | ||
tag.innerText = this.getSnippet(); | ||
this.AISKUPerfTest.snippetStartTime = performance.now(); | ||
document.getElementsByTagName("head")[0].appendChild(tag); | ||
} | ||
|
||
public getSnippet(): string { | ||
return ` | ||
loadSdkUsingRequire({ | ||
src: "https://js.monitor.azure.com/scripts/b/ai.${this.AISKUPerfTest.version}.min.js?${this.AISKUPerfTest.testPostfix}", | ||
onInit: function (appInsights) { | ||
console.log("snippet loaded"); | ||
appInsightsInitPerftest.loadScriptOnInit(); | ||
}, | ||
cfg: { | ||
instrumentationKey: "key", | ||
enablePerfMgr: true, | ||
maxBatchSizeInBytes: 1000000, | ||
maxBatchInterval: 30000000, | ||
extensionConfig: {} | ||
} | ||
});`; | ||
} | ||
|
||
public addPerfEvents() { | ||
this._addMemoryPerfEvent(this.initialMemoryUsage,"Configuration"); | ||
this._trackSingleEvent(); | ||
|
||
setTimeout(() =>{ | ||
this._trackBatchEvents(10); | ||
setTimeout(() =>{ | ||
this._trackBatchEvents(30); | ||
setTimeout(() =>{ | ||
this._trackBatchEvents(50); | ||
setTimeout(() =>{ | ||
this._trackBatchEvents(80); | ||
setTimeout(() =>{ | ||
this._trackBatchEvents(100); | ||
},1000); | ||
},1000); | ||
},1000); | ||
},1000); | ||
},1000); | ||
} | ||
|
||
public getScriptSrc(): string { | ||
return `https://js.monitor.azure.com/scripts/b/ai.${this.AISKUPerfTest.version}.min.js?${this.AISKUPerfTest.testPostfix}`; | ||
} | ||
|
||
private _addMemoryPerfEvent(initialMemoryUsage: number, metric?: string): void { | ||
let curMemory = performance["memory"]?.usedJSHeapSize; | ||
if (isNullOrUndefined(initialMemoryUsage) || initialMemoryUsage < 0) return; | ||
|
||
|
||
if (!isNullOrUndefined(curMemory)) { | ||
let metricName = metric? metric:"SDKInit"; | ||
var memoryUsed = Math.round((curMemory - initialMemoryUsage) / 1000); | ||
createPerfEvent(this.AISKUPerfTest, metricName, memoryUsed, false, `AppInsightsInit-Init: ${metricName} UsedJSHeapSize ${memoryUsed} KB added`); | ||
} | ||
} | ||
|
||
protected _trackSingleEvent() { | ||
if (isNullOrUndefined(this.appInsights)) return; | ||
|
||
var event = createTrackEvent("Track","EventData",20); | ||
var curMemeory = performance["memory"]?.usedJSHeapSize; | ||
if (! isNullOrUndefined(curMemeory)) { | ||
this.memoryUsageMarks["Track"] = curMemeory; | ||
} | ||
|
||
try { | ||
if (this.AISKUPerfTest.hasPerfMgr) { | ||
this.appInsights.trackEvent(event); | ||
} else { | ||
this.doPerf(this.perfMgrSrc, () => "AppInsightsCore:track", (perfEvent) => { | ||
this.appInsights.trackEvent(event); | ||
// A mark has been added to window.performance called 'tst.mark.AppInsightsCore:track' | ||
},()=> ({item: event})); | ||
} | ||
this.appInsights.flush(false); | ||
} catch (e) { | ||
console.error(e); | ||
} | ||
} | ||
|
||
protected _trackBatchEvents(number?: number) { | ||
var curTime = performance.now(); | ||
if (isNullOrUndefined(this.appInsights)) return; | ||
|
||
var eventNumber = number && number > 0? number:100; | ||
var eventName = "BatchSend" + number; | ||
this.batchStartTimeMarks[eventName] = curTime; | ||
var event = createTrackEvent("batch","Message",20); | ||
|
||
var curMemory = performance["memory"]?.usedJSHeapSize; | ||
if (!isNullOrUndefined(curMemory)) { | ||
this.memoryUsageMarks[eventName] = curMemory; | ||
} | ||
|
||
let beforeCreateBatch = performance.now(); | ||
try { | ||
for (let i = 0; i < eventNumber; i++) { | ||
this.appInsights.trackTrace(event); | ||
} | ||
let afterCreateBatch = performance.now(); | ||
let afterMemoUage = performance["memory"]?.usedJSHeapSize; | ||
let val = Math.round((afterMemoUage-curMemory)/1000); | ||
val = val < 0? 0:val; | ||
let msg = `${eventName} Memory Usage: ${val}KB`; | ||
createPerfEvent(this.AISKUPerfTest,eventName,val,false,msg); | ||
let duration = Math.round((afterCreateBatch-beforeCreateBatch)*1000)/1000; | ||
let createBatchName = `CreateBatch${eventNumber}` | ||
let createBatchMsg = `${createBatchName} time: ${duration}ms`; | ||
createPerfEvent(this.AISKUPerfTest,createBatchName,duration,true,createBatchMsg); | ||
if (this.AISKUPerfTest.hasPerfMgr) { | ||
this.appInsights.flush(false); | ||
} else { | ||
this.doPerf(this.perfMgrSrc, () => "AISKU.flush", (perfEvent) => { | ||
this.appInsights.flush(false); | ||
// A mark has been added to window.performance called 'tst.mark.AISKU.flush' | ||
},()=> ({item: event})); | ||
} | ||
} catch (e) { | ||
console.error(e); | ||
} | ||
|
||
} | ||
} |
Oops, something went wrong.