Task 26319951: [1ds][Post] Add support for the ext.metadata to NOT be…

… included (#2251)

channels/1ds-post-js/src/DataModels.ts

@@ -234,6 +234,17 @@ export interface IChannelConfiguration {
      * value to indicate that the server should return a 204 for successful requests. Defaults to true
      */
     addNoResponse?: boolean;
+
+    /**
+     * :warning: DO NOT USE THIS FLAG UNLESS YOU KNOW THAT PII DATA WILL NEVER BE INCLUDED IN THE EVENT!
+     *
+     * [Optional] Flag to indicate whether the SDK should include the common schema metadata in the payload. Defaults to true.
+     * This flag is only applicable to the POST channel and will cause the SDK to exclude the common schema metadata from the payload,
+     * while this will reduce the size of the payload, also means that the data marked as PII will not be processed as PII by the backend
+     * and will not be included in the PII data purge process.
+     * @since 4.1.0
+     */
+    excludeCsMetaData?: boolean;
 }
 
 /**

channels/1ds-post-js/src/HttpManager.ts

@@ -7,10 +7,10 @@ import dynamicProto from "@microsoft/dynamicproto-js";
 import {
     EventSendType, FullVersionString, IAppInsightsCore, ICookieMgr, IDiagnosticLogger, IExtendedConfiguration, IPayloadData, IPerfEvent,
     IUnloadHook, IXHROverride, OnCompleteCallback, SendPOSTFunction, SendRequestReason, TransportType, _eExtendedInternalMessageId,
-    _eInternalMessageId, _throwInternal, _warnToConsole, arrForEach, dateNow, doPerf, dumpObj, eLoggingSeverity, extend, getLocation,
-    getNavigator, getTime, hasOwnProperty, isArray, isBeaconsSupported, isFetchSupported, isNullOrUndefined, isNumber, isReactNative,
-    isString, isUndefined, isValueAssigned, isXhrSupported, objForEachKey, objKeys, onConfigChange, openXhr, strTrim, strUndefined,
-    useXDomainRequest
+    _eInternalMessageId, _throwInternal, _warnToConsole, arrForEach, dateNow, doPerf, dumpObj, eLoggingSeverity, extend,
+    getCommonSchemaMetaData, getLocation, getNavigator, getTime, hasOwnProperty, isArray, isBeaconsSupported, isFetchSupported,
+    isNullOrUndefined, isNumber, isReactNative, isString, isUndefined, isValueAssigned, isXhrSupported, objForEachKey, objKeys,
+    onConfigChange, openXhr, strTrim, strUndefined, useXDomainRequest
 } from "@microsoft/1ds-core-js";
 import { arrAppend } from "@nevware21/ts-utils";
 import { BatchNotificationAction, BatchNotificationActions } from "./BatchNotificationActions";
@@ -177,36 +177,40 @@ export class HttpManager {
      * @param requestQueue   - The queue that contains the requests to be sent.
      */
     constructor(maxEventsPerBatch: number, maxConnections: number, maxRequestRetriesBeforeBackoff: number, actions: BatchNotificationActions) {
+        // ------------------------------------------------------------------------------------------------------------------------
+        // Only set "Default" values in the _initDefaults() method, unless value are not "reset" during unloading
+        // ------------------------------------------------------------------------------------------------------------------------
         let _urlString: string;
-        let _killSwitch: KillSwitch = new KillSwitch();
-        let _paused = false;
-        let _clockSkewManager = new ClockSkewManager();
+        let _killSwitch: KillSwitch;
+        let _paused: boolean;
+        let _clockSkewManager: ClockSkewManager;
         let _useBeacons = false;
-        let _outstandingRequests = 0;           // Holds the number of outstanding async requests that have not returned a response yet
+        let _outstandingRequests: number;           // Holds the number of outstanding async requests that have not returned a response yet
         let _postManager: IPostChannel;
         let _logger: IDiagnosticLogger;
         let _sendInterfaces: { [key: number]: IInternalXhrOverride };
         let _core: IAppInsightsCore;
-        let _customHttpInterface = true;
-        let _queryStringParameters: IQueryStringParams[] = [];
-        let _headers: { [name: string]: string } = {};
-        let _batchQueue: EventBatch[] = [];
-        let _serializer: Serializer = null;
-        let _enableEventTimings = false;
+        let _customHttpInterface: boolean;
+        let _queryStringParameters: IQueryStringParams[];
+        let _headers: { [name: string]: string };
+        let _batchQueue: EventBatch[];
+        let _serializer: Serializer;
+        let _enableEventTimings: boolean;
         let _cookieMgr: ICookieMgr;
-        let _isUnloading = false;
-        let _useHeaders = false;
+        let _isUnloading: boolean;
+        let _useHeaders: boolean;
         let _xhrTimeout: number;
         let _disableXhrSync: boolean;
         let _disableFetchKeepAlive: boolean;
         let _canHaveReducedPayload: boolean;
         let _addNoResponse: boolean;
-        let _unloadHooks: IUnloadHook[]  = [];
+        let _unloadHooks: IUnloadHook[];
         let _sendHook: PayloadPreprocessorFunction | undefined;
         let _sendListener: PayloadListenerFunction | undefined;
-        let _responseHandlers: Array<(responseText: string) => void> = [];
+        let _responseHandlers: Array<(responseText: string) => void>;
         let _isInitialized: boolean;
         let _timeoutWrapper: ITimeoutOverrideWrapper;
+        let _excludeCsMetaData: boolean;
 
         dynamicProto(HttpManager, this, (_self) => {
             _initDefaults();
@@ -254,6 +258,7 @@ export class HttpManager {
                         _disableXhrSync = !!channelConfig.disableXhrSync;
                         _disableFetchKeepAlive = !!channelConfig.disableFetchKeepAlive;
                         _addNoResponse = channelConfig.addNoResponse !== false;
+                        _excludeCsMetaData = !!channelConfig.excludeCsMetaData;
 
 
                         if (!!core.getPlugin("LocalStorage")) {
@@ -262,7 +267,7 @@ export class HttpManager {
                         }
 
                         _useBeacons = !isReactNative(); // Only use beacons if not running in React Native
-                        _serializer = new Serializer(_core, valueSanitizer, stringifyObjects, enableCompoundKey);
+                        _serializer = new Serializer(_core, valueSanitizer, stringifyObjects, enableCompoundKey, getCommonSchemaMetaData, _excludeCsMetaData);
 
                         if (!isNullOrUndefined(channelConfig.useSendBeacon)) {
                             _useBeacons = !!channelConfig.useSendBeacon;
@@ -442,6 +447,7 @@ export class HttpManager {
                 _responseHandlers = [];
                 _isInitialized = false;
                 _timeoutWrapper = createTimeoutWrapper();
+                _excludeCsMetaData = false;
             }
 
             function _fetchSendPost(payload: IPayloadData, oncomplete: OnCompleteCallback, sync?: boolean) {

channels/1ds-post-js/src/PostChannel.ts

@@ -84,7 +84,8 @@ const defaultPostChannelConfig: IConfigDefaults<IChannelConfiguration> = objDeep
     alwaysUseXhrOverride: false,
     maxEventRetryAttempts: { isVal: isNumber, v: MaxSendAttempts },
     maxUnloadEventRetryAttempts: { isVal: isNumber, v: MaxSyncUnloadSendAttempts},
-    addNoResponse: undefValue
+    addNoResponse: undefValue,
+    excludeCsMetaData: undefValue
 });
 
 function isOverrideFn(httpXHROverride: any) {

channels/1ds-post-js/src/Serializer.ts

@@ -38,6 +38,30 @@ const metadata = "metadata";
 const f = "f";
 const rCheckDot = /\./;
 
+/**
+ * @ignore
+ * Identifies the callback to add metadata for a property.
+ * @since 4.1.0
+ * @group Private
+ * @param pathKeys - The path keys for the property
+ * @param key - The property key
+ * @param value - The property value
+ */
+type EventMetaDataCallback = (pathKeys: string[], key: string, value: IEventProperty) => void;
+
+/**
+ * @ignore
+ * Identifies the callback to get the encoded type for a property.
+ * This is added as a future hook for the serializer to allow for custom encoding of properties.
+ * @since 4.1.0
+ * @group Private
+ * @param value - The property value
+ * @param kind - The property value kind
+ * @param type - The property type
+ * @returns The encoded type for the property
+ */
+export type SerializerGetEncodedType = (value: string | boolean | number | string[] | number[] | boolean[] | undefined, kind: number | undefined, type?: number | undefined) => number;
+
 export interface ISerializedPayload {
     /**
      * The collection of iKeys included in this payload
@@ -116,15 +140,26 @@ export interface ISerializedPayload {
 */
 export class Serializer {
 
-    constructor(perfManager?: IPerfManagerProvider, valueSanitizer?: IValueSanitizer, stringifyObjects?: boolean, enableCompoundKey?: boolean) {
+    /**
+     * Constructs a new instance of the Serializer class
+     * @param perfManager - The performance manager to use for tracking performance
+     * @param valueSanitizer - The value sanitizer to use for sanitizing field values
+     * @param stringifyObjects - Should objects be stringified before being sent
+     * @param enableCompoundKey - Should compound keys be enabled (defaults to false)
+     * @param getEncodedTypeOverride - The callback to get the encoded type for a property defaults to ({@link getCommonSchemaMetaData }(...))
+     * @param excludeCsMetaData - (!DANGER!) Should metadata be populated when encoding the event blob (defaults to false) - PII data will NOT be tagged as PII for backend processing
+     */
+    constructor(perfManager?: IPerfManagerProvider, valueSanitizer?: IValueSanitizer, stringifyObjects?: boolean, enableCompoundKey?: boolean, getEncodedTypeOverride?: SerializerGetEncodedType, excludeCsMetaData?: boolean) {
         const strData = "data";
         const strBaseData = "baseData";
         const strExt = "ext";
 
         let _checkForCompoundkey = !!enableCompoundKey;
-        let _processSubMetaData = true;
+        let _processSubKeys = true;
         let _theSanitizer: IValueSanitizer = valueSanitizer;
         let _isReservedCache = {};
+        let _excludeCsMetaData: boolean = !!excludeCsMetaData;
+        let _getEncodedType: SerializerGetEncodedType = getEncodedTypeOverride || getCommonSchemaMetaData;
 
         dynamicProto(Serializer, this, (_self) => {
 
@@ -255,6 +290,13 @@ export class Serializer {
                         // Assigning local var so usage in part b/c don't throw if there is no ext
                         let serializedExt = {};
 
+                        let _addMetadataCallback: EventMetaDataCallback;
+                        if (!_excludeCsMetaData) {
+                            _addMetadataCallback = (pathKeys: string[], key: string, value: IEventProperty) => {
+                                _addJSONPropertyMetaData(_getEncodedType, serializedExt, pathKeys, key, value);
+                            };
+                        }
+
                         // Part A
                         let eventExt = eventData[strExt];
                         if (eventExt) {
@@ -274,14 +316,10 @@ export class Serializer {
                         let serializedBaseData = serializedData[strBaseData] = {};
 
                         // Part B
-                        _processPathKeys(eventData.baseData, serializedBaseData, strBaseData, false, [strBaseData], (pathKeys, name, value) => {
-                            _addJSONPropertyMetaData(serializedExt, pathKeys, name, value);
-                        }, _processSubMetaData);
+                        _processPathKeys(eventData.baseData, serializedBaseData, strBaseData, false, [strBaseData], _addMetadataCallback, _processSubKeys);
 
                         // Part C
-                        _processPathKeys(eventData.data, serializedData, strData, false, [], (pathKeys, name, value) => {
-                            _addJSONPropertyMetaData(serializedExt, pathKeys, name, value);
-                        }, _processSubMetaData);
+                        _processPathKeys(eventData.data, serializedData, strData, false, [], _addMetadataCallback, _processSubKeys);
 
                         return JSON.stringify(serializedEvent);
                     }, () => ({ item: eventData }));
@@ -311,7 +349,7 @@ export class Serializer {
                 thePath: string,
                 checkReserved: boolean,
                 metadataPathKeys: string[],
-                metadataCallback: (pathKeys: string[], key: string, value: IEventProperty) => void,
+                metadataCallback: EventMetaDataCallback,
                 processSubKeys: boolean) {
 
                 objForEachKey(srcObj, (key, srcValue) => {
@@ -443,10 +481,21 @@ export class Serializer {
 
 /**
  * @ignore
+ * @param getEncodedType - The function to get the encoded type for the property
+ * @param json - The json object to add the metadata to
+ * @param propKeys - The property keys to add to the metadata
+ * @param name - The name of the property
+ * @param propertyValue - The property value
  */
-function _addJSONPropertyMetaData(json: { [name: string]: {} }, propKeys: string[], name: string, propertyValue: IEventProperty | null) {
+function _addJSONPropertyMetaData(
+    getEncodedType: (value: string | boolean | number | string[] | number[] | boolean[] | undefined, kind: number | undefined, type?: number | undefined) => number,
+    json: { [name: string]: {} },
+    propKeys: string[],
+    name: string,
+    propertyValue: IEventProperty | null) {
+
     if (propertyValue && json) {
-        let encodedTypeValue = getCommonSchemaMetaData(propertyValue.value, propertyValue.kind, propertyValue.propertyType);
+        let encodedTypeValue = getEncodedType(propertyValue.value, propertyValue.kind, propertyValue.propertyType);
         if (encodedTypeValue > -1) {
             // Add the root metadata
             let metaData = json[metadata];

channels/1ds-post-js/test/Unit/src/PostChannelTest.ts

@@ -1,5 +1,5 @@
-import { AITestClass } from "@microsoft/ai-test-framework";
-import { IExtendedConfiguration, NotificationManager, AppInsightsCore, EventLatency, ITelemetryItem, IExtendedTelemetryItem, SendRequestReason, EventSendType, isFetchSupported, objKeys, arrForEach, isBeaconsSupported } from '@microsoft/1ds-core-js';
+import { AITestClass, TestHelper } from "@microsoft/ai-test-framework";
+import { IExtendedConfiguration, AppInsightsCore, EventLatency, ITelemetryItem, IExtendedTelemetryItem, SendRequestReason, EventSendType, isFetchSupported, objKeys, arrForEach, isBeaconsSupported, EventPersistence, isNullOrUndefined } from '@microsoft/1ds-core-js';
 import { PostChannel, IXHROverride, IPayloadData } from '../../../src/Index';
 import { IPostTransmissionTelemetryItem, IChannelConfiguration } from '../../../src/DataModels';
 import { SinonSpy } from 'sinon';
@@ -169,7 +169,8 @@ export class PostChannelTest extends AITestClass {
                     alwaysUseXhrOverride: false,
                     maxEventRetryAttempts: 6,
                     maxUnloadEventRetryAttempts: 2,
-                    addNoResponse: undefValue
+                    addNoResponse: undefValue,
+                    excludeCsMetaData: undefValue
                 };
                 let actaulConfig =  postChannel["_getDbgPlgTargets"]()[1];
                 QUnit.assert.deepEqual(expectedConfig, actaulConfig, "default config should be set");
@@ -3615,6 +3616,59 @@ export class PostChannelTest extends AITestClass {
 
             }
         });
+
+        this.testCase({
+            name: "Post Channel: check excludeCsMetaData",
+            useFakeTimers: true,
+            useFakeServer: true,
+            test: () => {
+                let config = this.config;
+                let core = this.core;
+                let postChannel = this.postChannel;
+                let identifier = postChannel.identifier;
+                let event1: IPostTransmissionTelemetryItem = TestHelper.mockEvent(EventPersistence.Normal);
+                let event2: IPostTransmissionTelemetryItem = TestHelper.mockEvent(EventPersistence.Normal);
+                core.initialize(config, [postChannel]);
+
+                // test timeout
+                core.track(event1);
+                this.clock.tick(10001);
+
+                let requests = this._getXhrRequests();
+                QUnit.assert.equal(requests.length, 1, "request should be sent");
+                requests[0].respond(200, {}, "response body");
+
+                QUnit.assert.equal(requests[0].method, "POST", "request method should be POST");
+
+                let evt = JSON.parse(requests[0]["requestBody"]);
+                QUnit.assert.equal(evt.name, event1.name, "request data should be set");
+                let metaData = evt.ext.metadata;
+                QUnit.assert.ok(!isNullOrUndefined(metaData));
+                QUnit.assert.equal(metaData.f.evValue2.t, 8193, "evValue should be tagged");
+                QUnit.assert.equal(metaData.f.value5.t, 6, "value5 should be tagged as number");
+                QUnit.assert.equal(metaData.f.value1.a.t, 6, "value1 should be tagged as array of numbers");
+
+                // Disable CsMetaData
+                core!.config!.extensionConfig![identifier].excludeCsMetaData = true;
+                // Let the dynamic config changes occur
+                this.clock.tick(1);
+
+                core.track(event2);
+                this.clock.tick(10001);
+
+                requests = this._getXhrRequests();
+                QUnit.assert.equal(requests.length, 2, "request should be sent");
+                requests[1].respond(200, {}, "response body");
+
+                QUnit.assert.equal(requests[1].method, "POST", "request method should be POST");
+
+                evt = JSON.parse(requests[1]["requestBody"]);
+                QUnit.assert.equal(evt.name, event2.name, "request data should be set");
+                metaData = evt.ext.metadata;
+                QUnit.assert.equal(metaData, undefined, "metadata should be undefined");
+            }
+        });
+
     }
 }