Merge pull request #2 from Azure/docs

improving docs

azure-kusto-data/README.md

@@ -1,19 +1,63 @@
 # Microsoft Azure Kusto Data Library for Node
 
-# Quick Start
+## Installation
+
+`npm install azure-kusto-data`
+
+## Quick Start
 
 ```javascript 
 const KustoClient = require("azure-kusto-data").Client;
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
-const Console = require("console");
 
-const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password','tenant_id');
+const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://${clusterName}.kusto.windows.net`,'appid','appkey','authorityId');
 const client = new KustoClient(kcsb);
 
 client.execute("db", "TableName | limit 1", (err, results) => {
     if (err) throw new Error(err);
-    Console.log(JSON.stringify(results));
-    Console.log(results.primaryResults[0].toString());
+    console.log(JSON.stringify(results));
+    console.log(results.primaryResults[0].toString());
 });
 
-```
+```
+
+## Authentication
+There are several authentication methods
+
+### AAD appliction
+There are two ways to authenticate using AAD application:
+Option 1: Authenticating using AAD application id and corresponding key.
+```javascript
+const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://${clusterName}.kusto.windows.net`,'appid','appkey','authorityId');
+```
+
+Option 2: Authenticating using AAD application id and corresponding certificate.
+
+```javascript
+const kcsb = KustoConnectionStringBuilder.withAadApplicationCertificateAuthentication(`https://${clusterName}.kusto.windows.net`, 'appid', 'certificate', 'thumbprint', 'authorityId');
+```
+
+
+### Username/Password
+```javascript
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password');
+```
+
+Authority is optional *when it can inferred from the domain* ('user@microsoft.com' would make the authority 'microsoft.com'). 
+In any case it is possible to pass the authority id
+```javascript
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password','authorityId');
+```
+
+### Device
+Using this method will write a token to the console, which can be used to authenticate at https://login.microsoftonline.com/common/oauth2/deviceauth and will allow temporary access. 
+
+**<!>It is not meant for production purposes<!>**
+
+```javascript
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password');
+```
+
+## Usage
+Query language docs can be found at https://docs.microsoft.com/en-us/azure/data-explorer/write-queries#overview-of-the-query-language
+

azure-kusto-data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "azure-kusto-data",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "",
   "main": "index.js",
   "engines": {

azure-kusto-ingest/README.md

@@ -1,24 +1,33 @@
-# Microsoft Azure Kusto Data Library for Node
+# Microsoft Azure Kusto Ingest Library for Node
 
-# Quick Start
+## Installation
+
+`npm install azure-kusto-ingest`
+
+## Quick Start
 
 ```javascript 
 const IngestClient = require("azure-kusto-ingest").IngestClient;
 const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
 const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
 const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest").IngestionPropertiesEnums;
 
-const ingestClient = new IngestClient(
-    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${cluster}.kusto.windows.net`, appId, appKey, tenantId),
-    new IngestionProps(
+const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${cluster}.kusto.windows.net`, appId, appKey, authorityId);
+
+const ingestionProps = new IngestionProps(
         "Database",
         "Table",
         DataFormat.json,
         [
             new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
             new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
             new JsonColumnMapping("TargetColumn3", "$.sourceProp3")
-        ])
+        ]
+);
+
+const ingestClient = new IngestClient(
+    kcsb,
+    ingestionProps
 );
 
 console.log("Ingest from file");
@@ -31,4 +40,220 @@ ingestClient.ingestFromFile("file.json", null, (err) => {
     console.log("Ingestion done");
 });
 
-```
+```
+
+## Authentication
+There are several authentication methods
+
+### AAD App
+The are two ways to authenticate is to use app id and key
+
+1. Using app key
+```javascript
+const kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'appid','appkey','authorityId');
+```
+
+1. Using a certificate:
+
+```javascript
+const kcsb = KustoConnectionStringBuilder.withAadApplicationCertificateAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, 'appid', 'certificate', 'thumbprint', 'authorityId');
+```
+
+
+### Username/Password
+```javascript
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password');
+```
+
+Authority is optional *when it can be inferred from the domain* ('user@microsoft.com' would make the authority 'microsoft.com'). 
+In any case it is possible to pass the authority id
+```javascript
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'username','password','authorityId');
+```
+
+### Device
+Using this method will write a token to the console, which can be used to authenticate at https://login.microsoftonline.com/common/oauth2/deviceauth and will allow temporary access. 
+
+**<!>It is not ment for production purposes<!>**
+
+```javascript
+KustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'username','password');
+```
+
+## Usage
+
+A Quick Overview is available at https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview
+
+Notice ingestion is done against the ingestion endpoint, which usually include `ingest-` prefix on the cluster name.
+
+### Ingestion Properties
+Ingestion Props are instructions for Kusto on how to process the data.
+
+The easiest way to provide ingestion properties is to set them on the ingestion client like in the sample above.
+It is also possible to pass them on each ingestion (will merge them with default props).
+
+Example props:
+
+```javascript
+const ingestionProps = new IngestionProps(
+        "Database",
+        "Table",
+        DataFormat.json,
+        [
+            new JsonColumnMapping("TargetColumn1", "$.sourceProp1"),
+            new JsonColumnMapping("TargetColumn2", "$.sourceProp2"),
+            new JsonColumnMapping("TargetColumn3", "$.sourceProp3")
+        ]
+);
+```
+
+### Ingestion Sources
+There are several methods of ingesting data into Kusto (Azure Data Explorer) using this library
+
+#### From Stream
+
+This is useful for cases you already have streams available (http respinse, file stream, etc..)
+
+```javascript
+ingestClient.ingestFromStream(readable, null, (err) => {
+        if (err) console.log(err);
+        else console.log("Ingestion from stream DONE");
+});
+```
+
+
+#### From File
+
+Ingesting a file first makes sure it's zipped (if not, it zips it locally) and then send it for ingestion
+
+```javascript
+ingestClient.ingestFromFile(filePath, null, (err) => {
+        if (err) {
+            console.log(err);
+        }
+
+        console.log("Ingestion from file DONE");
+
+
+        setTimeout(waitForFailures, 0);
+        setTimeout(waitForSuccess, 0);
+    });
+```
+
+#### From Azure Storage Blob
+
+Probably the easiest way would be to provide a uri (with [SAS](https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1)).
+
+```javascript
+
+let blob = new BlobDescriptor(blobUri, size);
+ingestClient.ingestFromBlob(blob, null, (err) => {
+        if (err) {
+            console.log(err);
+        }
+
+        console.log("Ingestion from file DONE");
+
+
+        setTimeout(waitForFailures, 0);
+        setTimeout(waitForSuccess, 0);
+    });
+```
+
+### Ingestion Status
+
+It is possible to track the status of the ingestion using status queues.
+
+Enabling is done simply but setting the `reportLevel` Ingestion Property to `ReportLevel.FailuresAndSuccesses`
+
+For Example:
+
+```javascript
+const IngestClient = require("azure-kusto-ingest").IngestClient;
+const IngestStatusQueues = require("azure-kusto-ingest").IngestStatusQueues;
+const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
+const { ReportLevel, ReportMethod } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
+const { DataFormat, JsonColumnMapping } = require("azure-kusto-ingest").IngestionPropertiesEnums;
+const fs = require("fs");
+
+
+const ingestClient = new IngestClient(
+    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, appId, appKey, authorityId),
+    new IngestionProps(
+        "db",
+        "table",
+        DataFormat.json,
+        [
+            new JsonColumnMapping("Id", "$.id"),
+            new JsonColumnMapping("Type", "$.type"),
+            new JsonColumnMapping("Value", "$.type"),
+        ],
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,      
+        ReportLevel.FailuresAndSuccesses,
+        ReportMethod.Queue)
+);
+
+const statusQueues = new IngestStatusQueues(ingestClient);
+
+function waitForFailures() {
+    statusQueues.failure.isEmpty((err, empty) => {
+        if (err) throw new Error(err);
+
+        if (empty) {
+            console.log("no errors...");
+            return setTimeout(waitForFailures, 1000);
+        }
+        else {
+            statusQueues.failure.pop((err, failures) => {
+                if (err) throw new Error(err);
+
+                for (let failure of failures) {
+                    console.log(JSON.stringify(failure));
+                }
+
+                return setTimeout(waitForFailures, 1000);
+            });
+        }
+    });
+}
+
+function waitForSuccess() {
+    statusQueues.success.isEmpty((err, empty) => {
+        if (err) throw new Error(err);
+
+        if (empty) {
+            console.log("no successes...");
+            return setTimeout(waitForSuccess, 1000);
+        }
+        else {
+            statusQueues.success.pop((err, successes) => {
+                if (err) throw new Error(err);
+
+                for (let success of successes) {
+                    console.log(JSON.stringify(success));
+                }
+
+                return setTimeout(waitForSuccess, 1000);
+            })
+        }
+    });
+}
+
+ingestClient.ingestFromFile("file.json", null, (err) => {
+    if (err) {
+        console.log(err);
+    }
+
+    console.log("Ingestion done?");
+
+
+    setTimeout(waitForFailures, 0);
+    setTimeout(waitForSuccess, 0);
+});
+```

azure-kusto-ingest/package.json

@@ -1,6 +1,6 @@
 {
   "name": "azure-kusto-ingest",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "",
   "main": "./index.js",
   "engines": {