T shanim/enable multiple queries

src/generators.ts

@@ -1,6 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import { ConnectionOnlyQueryInfo, QueryInfo } from "./types";
+import { missingQueryNameErr } from "./utils/constants";
+
 export const generateMashupXMLTemplate = (base64: string): string =>
     `<?xml version="1.0" encoding="utf-16"?><DataMashup xmlns="http://schemas.microsoft.com/DataMashup">${base64}</DataMashup>`;
 
@@ -10,4 +13,25 @@ export const generateSingleQueryMashup = (queryName: string, query: string): str
     shared #"${queryName}" = 
     ${query};`;
 
+export const generateMultipleQueryMashup = (loadedQuery: QueryInfo, queries: ConnectionOnlyQueryInfo[]): string => {
+    if (!loadedQuery.queryName) {
+        throw new Error(missingQueryNameErr);
+    }
+
+    let mashup: string = generateSingleQueryMashup(loadedQuery.queryName, loadedQuery.queryMashup);
+    queries.forEach((query: ConnectionOnlyQueryInfo) => {
+        const queryName = query.queryName;
+        if (!queryName) {
+            throw new Error(missingQueryNameErr);
+        }
+
+        mashup += `
+
+        shared #"${queryName}" = 
+        ${query.queryMashup};`;
+    });
+
+    return mashup;
+}
+
 export const generateCustomXmlFilePath = (i: number): string => `customXml/item${i}.xml`;

src/types.ts

@@ -7,6 +7,13 @@ export interface QueryInfo {
     queryName?: string;
 }
 
+export interface QueriesInfo {
+    loadedQuery: QueryInfo;
+    connectionOnlyQueries: ConnectionOnlyQueryInfo[];
+}
+
+export type ConnectionOnlyQueryInfo = Omit<QueryInfo, 'refreshOnOpen'>;
+
 export interface DocProps {
     title?: string | null;
     subject?: string | null;

src/utils/constants.ts

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import { v4 } from "uuid";
+
 export const connectionsXmlPath = "xl/connections.xml";
 export const sharedStringsXmlPath = "xl/sharedStrings.xml";
 export const sheetsXmlPath = "xl/worksheets/sheet1.xml";
@@ -40,6 +42,8 @@ export const unexpectedErr = "Unexpected error";
 export const arrayIsntMxNErr = "Array isn't MxN";
 export const templateFileNotSupportedErr = "Template file is not supported for this API call";
 export const relsNotFoundErr = ".rels were not found in template";
+export const queryNameAlreadyExistsErr = "Queries must have unique names";
+export const missingQueryNameErr = "Query name is missing";
 
 export const blobFileType = "blob";
 export const uint8ArrayType = "uint8array";
@@ -85,6 +89,9 @@ export const element = {
     dimension: "dimension",
     selection: "selection",
     kindCell: "c",
+    connection: "connection",
+    connections: "connections",
+    databaseProps: "dbPr",
 };
 
 export const elementAttributes = {
@@ -99,6 +106,7 @@ export const elementAttributes = {
     name: "name",
     description: "description",
     id: "id",
+    typeLowerCase: "type",
     type: "Type",
     value: "Value",
     relationshipInfo: "RelationshipInfoContainer",
@@ -117,6 +125,19 @@ export const elementAttributes = {
     spans: "spans",
     x14acDyDescent: "x14ac:dyDescent",
     xr3uid: "xr3:uid",
+    xr16uid: "xr16:uid",
+    keepAlive: "keepAlive",
+    refreshedVersion: "refreshedVersion",
+    background: "background",
+    isPrivate: "IsPrivate",
+    fillEnabled: "FillEnabled",
+    fillObjectType: "FillObjectType",
+    fillToDataModelEnabled: "FillToDataModelEnabled",
+    filLastUpdated: "FillLastUpdated",
+    filledCompleteResultToWorksheet: "FilledCompleteResultToWorksheet",
+    addedToDataModel: "AddedToDataModel",
+    fillErrorCode: "FillErrorCode",
+    fillStatus: "FillStatus",
 };
 
 export const dataTypeKind = {
@@ -125,16 +146,27 @@ export const dataTypeKind = {
     boolean: "b",
 };
 
+export const powerQueryResultType = {
+    table: "sTable",
+    connectionOnly: "sConnectionOnly",
+};
+
+export const itemPathTextContext = (queryName: string, isSource: boolean) => isSource ? `Section1/${encodeURIComponent(queryName)}/Source` : `Section1/${queryName}`;
+
 export const elementAttributesValues = {
     connectionName: (queryName: string) => `Query - ${queryName}`,
     connectionDescription: (queryName: string) => `Connection to the '${queryName}' query in the workbook.`,
     connection: (queryName: string) => `Provider=Microsoft.Mashup.OleDb.1;Data Source=$Workbook$;Location="${queryName}";`,
     connectionCommand: (queryName: string) => `SELECT * FROM [${queryName}]`,
-    tableResultType: () => "sTable",
+    fillStatusComplete: () => "sComplete",
+    fillErrorCodeUnknown: () => "sUnknown",
+    randomizedUid: () => "{" + v4().toUpperCase() + "}",
+    defaultConnectionType: () => "5",
 };
 
 export const defaults = {
     queryName: "Query1",
+    connectionOnlyQueryNamePrefix: "Connection only query-",
     sheetName: "Sheet1",
     columnName: "Column",
 };

src/utils/mashupDocumentParser.ts

@@ -16,6 +16,8 @@ import {
     divider,
     elementAttributes,
     elementAttributesValues,
+    itemPathTextContext,
+    powerQueryResultType,
 } from "./constants";
 import { arrayUtils } from ".";
 import { Metadata } from "../types";
@@ -42,6 +44,14 @@ export const replaceSingleQuery = async (base64Str: string, queryName: string, q
     return base64.fromByteArray(newMashup);
 };
 
+export const addConnectionOnlyQueries = async (base64Str: string, connectionOnlyQueryNames: string[]): Promise<string> => {
+        var { version, packageOPC, permissionsSize, permissions, metadata, endBuffer } = getPackageComponents(base64Str);
+        const newMetadataBuffer: Uint8Array = addConnectionOnlyQuerieMetadata(metadata, connectionOnlyQueryNames);
+        const newMashup: Uint8Array = arrayUtils.concatArrays(version, arrayUtils.getInt32Buffer(packageOPC.byteLength), packageOPC, arrayUtils.getInt32Buffer(permissionsSize), permissions, arrayUtils.getInt32Buffer(newMetadataBuffer.byteLength), newMetadataBuffer, endBuffer);
+
+        return base64.fromByteArray(newMashup);
+};
+
 type PackageComponents = {
     version: Uint8Array;
     packageOPC: Uint8Array;
@@ -131,7 +141,7 @@ export const editSingleQueryMetadata = (metadataArray: Uint8Array, metadata: Met
                 return prop?.name === elementAttributes.type;
             });
             if (entryProp?.nodeValue == elementAttributes.resultType) {
-                entry.setAttribute(elementAttributes.value, elementAttributesValues.tableResultType());
+                entry.setAttribute(elementAttributes.value, powerQueryResultType.table);
             }
 
             if (entryProp?.nodeValue == elementAttributes.fillLastUpdated) {
@@ -150,3 +160,93 @@ export const editSingleQueryMetadata = (metadataArray: Uint8Array, metadata: Met
 
     return newMetadataArray;
 };
+
+const addConnectionOnlyQuerieMetadata = (metadataArray: Uint8Array, connectionOnlyQueryNames: string[]) => {
+        // extract metadataXml
+        const mashupArray: ArrayReader = new arrayUtils.ArrayReader(metadataArray.buffer);
+        const metadataVersion: Uint8Array = mashupArray.getBytes(4);
+        const metadataXmlSize: number = mashupArray.getInt32();
+        const metadataXml: Uint8Array = mashupArray.getBytes(metadataXmlSize);
+        const endBuffer: Uint8Array = mashupArray.getBytes();
+
+        // parse metadataXml
+        const metadataString: string = new TextDecoder("utf-8").decode(metadataXml);
+        const newMetadataString: string = addConnectionOnlyQueriesToMetadataStr(metadataString, connectionOnlyQueryNames);
+        const encoder: TextEncoder = new TextEncoder();
+        const newMetadataXml: Uint8Array = encoder.encode(newMetadataString);
+        const newMetadataXmlSize: Uint8Array = arrayUtils.getInt32Buffer(newMetadataXml.byteLength);
+        const newMetadataArray: Uint8Array = arrayUtils.concatArrays(
+            metadataVersion,
+            newMetadataXmlSize,
+            newMetadataXml,
+            endBuffer
+        );
+
+        return newMetadataArray;
+    };
+
+export const addConnectionOnlyQueriesToMetadataStr = (metadataString: string, connectionOnlyQueryNames: string[]) => {
+    const parser: DOMParser = new DOMParser();
+    let metadataDoc: Document = parser.parseFromString(metadataString, xmlTextResultType);      
+    connectionOnlyQueryNames.forEach((queryName: string) => {
+        const items: Element = metadataDoc.getElementsByTagName(element.items)[0];
+        const stableEntriesItem: Element = createStableEntriesItem(metadataDoc, queryName);
+        items.appendChild(stableEntriesItem);
+        const sourceItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item);
+        sourceItem.appendChild(createItemLocation(metadataDoc, queryName, true));
+        const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries);
+        sourceItem.appendChild(stableEntries);
+        items.appendChild(sourceItem);
+    });    
+
+        const updatedMetdataString: string = new XMLSerializer().serializeToString(metadataDoc);
+
+        return updatedMetdataString;
+    };
+
+    const createItemLocation = (metadataDoc: Document, queryName: string, isSource: boolean) => {
+        const newItemLocation: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemLocation);
+        const newItemType: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemType);
+        newItemType.textContent = "Formula";
+        const newItemPath: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.itemPath);
+        newItemPath.textContent = itemPathTextContext(queryName, isSource);
+        newItemLocation.appendChild(newItemType);
+        newItemLocation.appendChild(newItemPath);        
+
+        return newItemLocation;
+    };
+
+    const createStableEntriesItem = (metadataDoc: Document, queryName: string) => {
+        const newItem: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.item);
+        newItem.appendChild(createItemLocation(metadataDoc, queryName, false));
+        const stableEntries: Element = createEntries(metadataDoc, powerQueryResultType.connectionOnly);
+        newItem.appendChild(stableEntries);
+
+        return newItem;
+    };
+
+    const createElementObject = (metadataDoc: Document, type: string, value: string) => {
+        const elementObject: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.entry);
+        elementObject.setAttribute(elementAttributes.type, type);
+        elementObject.setAttribute(elementAttributes.value, value);
+
+        return elementObject;
+    };
+
+    const createEntries = (metadataDoc: Document, fillObjectType: string) => {
+        const stableEntries: Element = metadataDoc.createElementNS(metadataDoc.documentElement.namespaceURI, element.stableEntries);
+        const nowTime: string = new Date().toISOString();
+
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.isPrivate, "l0"));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillEnabled, "l0"));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillObjectType, fillObjectType));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillToDataModelEnabled, "l0"));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillLastUpdated, (elementAttributes.day + nowTime).replace(/Z/, "0000Z")));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.resultType, powerQueryResultType.table));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.filledCompleteResultToWorksheet, "l0"));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.addedToDataModel, "l0"));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillErrorCode, elementAttributesValues.fillErrorCodeUnknown()));
+        stableEntries.appendChild(createElementObject(metadataDoc, elementAttributes.fillStatus, elementAttributesValues.fillStatusComplete()));
+
+        return stableEntries;
+    };

src/utils/pqUtils.ts

@@ -2,9 +2,10 @@
 // Licensed under the MIT license.
 
 import JSZip from "jszip";
-import { EmptyQueryNameErr, QueryNameMaxLengthErr, maxQueryLength, URLS, BOM, QueryNameInvalidCharsErr } from "./constants";
+import { EmptyQueryNameErr, QueryNameMaxLengthErr, maxQueryLength, URLS, BOM, QueryNameInvalidCharsErr, queryNameAlreadyExistsErr, defaults } from "./constants";
 import { generateMashupXMLTemplate, generateCustomXmlFilePath } from "../generators";
 import { Buffer } from "buffer";
+import { ConnectionOnlyQueryInfo } from "../types";
 
 type CustomXmlFile = {
     found: boolean;
@@ -110,10 +111,56 @@ const validateQueryName = (queryName: string): void => {
         throw new Error(EmptyQueryNameErr);
     }
 };
+
+const validateMultipleQueryNames = (queries: ConnectionOnlyQueryInfo[],  loadedQueryName: string): string[] => {
+    const queryNames: string[] = [];
+    const cleanedLoadedQueryName: string = loadedQueryName.trim().toLowerCase();
+    queries.forEach((query: ConnectionOnlyQueryInfo) => {
+        if (query.queryName) {
+            validateQueryName(query.queryName);
+            const cleanedQueryName: string | undefined = query.queryName.trim().toLowerCase();
+            if (queryNames.includes(cleanedQueryName) || cleanedQueryName === cleanedLoadedQueryName) {
+                throw new Error(queryNameAlreadyExistsErr);
+            }
+
+            queryNames.push(cleanedQueryName);
+        } 
+    });
+
+    return queryNames;
+};
+
+const assignQueryNames = (queries: ConnectionOnlyQueryInfo[], loadedQueryName: string, queryNames: string[]):  ConnectionOnlyQueryInfo[] => {
+    // Generate unique name for queries without a name
+    queries.forEach((query: ConnectionOnlyQueryInfo) => {
+        if (!query.queryName) {
+            query.queryName = generateUniqueQueryName(queryNames, loadedQueryName);
+            queryNames.push(query.queryName.toLowerCase());
+        }
+    });
+
+    return queries;
+};
+
+const generateUniqueQueryName = (queryNames: string[], loadedQueryName: string,): string => {
+    let index: number = 1;
+    let queryName: string = defaults.connectionOnlyQueryNamePrefix + index++;
+    const cleanedLoadedQueryName: string = loadedQueryName.trim().toLowerCase();
+    // Assumes that query names are lower case
+    while (queryNames.includes(queryName.toLowerCase()) || queryName.toLowerCase() === cleanedLoadedQueryName) {
+        queryName = defaults.connectionOnlyQueryNamePrefix + index++;
+    }
+
+    return queryName;
+};
+
 export default {
     getBase64,
     setBase64,
     getCustomXmlFile,
     getDataMashupFile,
     validateQueryName,
+    assignQueryNames,
+    validateMultipleQueryNames,
+    generateUniqueQueryName
 };

src/utils/xmlInnerPartsUtils.ts

@@ -268,6 +268,27 @@ const updatePivotTable = (tableXmlString: string, connectionId: string, refreshO
     return { isPivotTableUpdated, newPivotTable };
 };
 
+const addNewConnection = async (connectionsDoc: Document, queryName: string): Promise<Document> => {
+    const connections = connectionsDoc.getElementsByTagName(element.connections)[0];
+    const newConnection = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.connection);
+    newConnection.setAttribute(elementAttributes.id, [...connectionsDoc.getElementsByTagName(element.connection)].length.toString());
+    newConnection.setAttribute(elementAttributes.xr16uid, elementAttributesValues.randomizedUid());
+    newConnection.setAttribute(elementAttributes.keepAlive, trueValue);
+    newConnection.setAttribute(elementAttributes.name, elementAttributesValues.connectionName(queryName));
+    newConnection.setAttribute(elementAttributes.description, elementAttributesValues.connectionDescription(queryName));
+    newConnection.setAttribute(elementAttributes.typeLowerCase, elementAttributesValues.defaultConnectionType());
+    newConnection.setAttribute(elementAttributes.refreshedVersion, falseValue);
+    newConnection.setAttribute(elementAttributes.background, trueValue);
+    connections.append(newConnection);
+
+    const databaseProperties = connectionsDoc.createElementNS(connectionsDoc.documentElement.namespaceURI, element.databaseProps);
+    databaseProperties.setAttribute(elementAttributes.connection, elementAttributesValues.connection(queryName));
+    databaseProperties.setAttribute(elementAttributes.command, elementAttributesValues.connectionCommand(queryName));
+    newConnection.appendChild(databaseProperties);
+
+    return connectionsDoc;
+    };
+
 export default {
     updateDocProps,
     clearLabelInfo,
@@ -277,4 +298,5 @@ export default {
     updatePivotTablesandQueryTables,
     updateQueryTable,
     updatePivotTable,
+    addNewConnection,
 };