diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
index cc03c5583c..e52f9d2014 100644
--- a/Microsoft.ML.sln
+++ b/Microsoft.ML.sln
@@ -141,6 +141,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML", "src\Microso
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Mkl.Redist", "src\Microsoft.ML.Mkl.Redist\Microsoft.ML.Mkl.Redist.csproj", "{4584326B-C5B3-4CAE-B98A-34C5F5AA16F3}"
EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis", "src\Microsoft.Data.Analysis\Microsoft.Data.Analysis.csproj", "{84150C22-0627-4A11-81C9-F214762855EA}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.Interactive", "src\Microsoft.Data.Analysis.Interactive\Microsoft.Data.Analysis.Interactive.csproj", "{D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.Tests", "test\Microsoft.Data.Analysis.Tests\Microsoft.Data.Analysis.Tests.csproj", "{0B765344-11A4-4738-9759-5060599DC134}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.Interactive.Tests", "test\Microsoft.Data.Analysis.Interactive.Tests\Microsoft.Data.Analysis.Interactive.Tests.csproj", "{8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -1680,6 +1688,102 @@ Global
{4584326B-C5B3-4CAE-B98A-34C5F5AA16F3}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
{4584326B-C5B3-4CAE-B98A-34C5F5AA16F3}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
{4584326B-C5B3-4CAE-B98A-34C5F5AA16F3}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug|x64.Build.0 = Debug|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netcoreapp3_1|Any CPU.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netcoreapp3_1|Any CPU.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netcoreapp3_1|x64.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netcoreapp3_1|x64.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release|Any CPU.Build.0 = Release|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release|x64.ActiveCfg = Release|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release|x64.Build.0 = Release|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netcoreapp3_1|Any CPU.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netcoreapp3_1|Any CPU.Build.0 = Release-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netcoreapp3_1|x64.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netcoreapp3_1|x64.Build.0 = Release-netcoreapp3_1|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+ {84150C22-0627-4A11-81C9-F214762855EA}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug|x64.Build.0 = Debug|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netcoreapp3_1|Any CPU.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netcoreapp3_1|Any CPU.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netcoreapp3_1|x64.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netcoreapp3_1|x64.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release|Any CPU.Build.0 = Release|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release|x64.ActiveCfg = Release|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release|x64.Build.0 = Release|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netcoreapp3_1|Any CPU.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netcoreapp3_1|Any CPU.Build.0 = Release-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netcoreapp3_1|x64.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netcoreapp3_1|x64.Build.0 = Release-netcoreapp3_1|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug|x64.Build.0 = Debug|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netcoreapp3_1|Any CPU.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netcoreapp3_1|Any CPU.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netcoreapp3_1|x64.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netcoreapp3_1|x64.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release|Any CPU.Build.0 = Release|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release|x64.ActiveCfg = Release|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release|x64.Build.0 = Release|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netcoreapp3_1|Any CPU.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netcoreapp3_1|Any CPU.Build.0 = Release-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netcoreapp3_1|x64.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netcoreapp3_1|x64.Build.0 = Release-netcoreapp3_1|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+ {0B765344-11A4-4738-9759-5060599DC134}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug|x64.Build.0 = Debug|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netcoreapp3_1|Any CPU.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netcoreapp3_1|Any CPU.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netcoreapp3_1|x64.ActiveCfg = Debug-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netcoreapp3_1|x64.Build.0 = Debug-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release|Any CPU.Build.0 = Release|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release|x64.ActiveCfg = Release|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release|x64.Build.0 = Release|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netcoreapp3_1|Any CPU.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netcoreapp3_1|Any CPU.Build.0 = Release-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netcoreapp3_1|x64.ActiveCfg = Release-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netcoreapp3_1|x64.Build.0 = Release-netcoreapp3_1|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -1749,6 +1853,10 @@ Global
{C8DB58DC-6434-4431-A81F-263D86E2A5F3} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
{6CF88209-69DB-4B36-9604-3ECD9F163E96} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
{4584326B-C5B3-4CAE-B98A-34C5F5AA16F3} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+ {84150C22-0627-4A11-81C9-F214762855EA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+ {D9FDD2D5-BFFC-4A4D-8589-7F63AA3EA923} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+ {0B765344-11A4-4738-9759-5060599DC134} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+ {8AFB8CC3-DA0B-4364-BFB3-296A7C54CC25} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
diff --git a/NuGet.config b/NuGet.config
index 5712c11fff..231d0f8008 100644
--- a/NuGet.config
+++ b/NuGet.config
@@ -11,7 +11,7 @@
-
+
diff --git a/eng/Versions.props b/eng/Versions.props
index 353e9fecb6..2e24673209 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -32,8 +32,11 @@
2.3.1
2
0.20.1
-
3.3.1
+ 1.0.0-beta.20410.1
+ 1.0.0-beta.20410.1
+ 2.0.0
+ 4.3.0
4.5.0
1.2.0
diff --git a/src/Microsoft.Data.Analysis.Interactive/DataFrameKernelExtension.cs b/src/Microsoft.Data.Analysis.Interactive/DataFrameKernelExtension.cs
new file mode 100644
index 0000000000..595c71e5a6
--- /dev/null
+++ b/src/Microsoft.Data.Analysis.Interactive/DataFrameKernelExtension.cs
@@ -0,0 +1,155 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Html;
+using Microsoft.DotNet.Interactive;
+using Microsoft.DotNet.Interactive.Formatting;
+using static Microsoft.DotNet.Interactive.Formatting.PocketViewTags;
+
+namespace Microsoft.Data.Analysis.Interactive
+{
+ public class DataFrameKernelExtension : IKernelExtension
+ {
+ public Task OnLoadAsync(Kernel kernel)
+ {
+ RegisterDataFrame();
+
+ return Task.CompletedTask;
+ }
+
+ public static void RegisterDataFrame()
+ {
+ Formatter.Register((df, writer) =>
+ {
+ const int MAX = 10000;
+ const int SIZE = 10;
+
+ var uniqueId = DateTime.Now.Ticks;
+
+ var header = new List
+ {
+ th(i("index"))
+ };
+ header.AddRange(df.Columns.Select(c => (IHtmlContent)th(c.Name)));
+
+ if (df.Rows.Count > SIZE)
+ {
+ var maxMessage = df.Rows.Count > MAX ? $" (showing a max of {MAX} rows)" : string.Empty;
+ var title = h3[style: "text-align: center;"]($"DataFrame - {df.Rows.Count} rows {maxMessage}");
+
+ // table body
+ var maxRows = Math.Min(MAX, df.Rows.Count);
+ var rows = new List>();
+ for (var index = 0; index < maxRows; index++)
+ {
+ var cells = new List
+ {
+ td(i((index)))
+ };
+ foreach (var obj in df.Rows[index])
+ {
+ cells.Add(td(obj));
+ }
+ rows.Add(cells);
+ }
+
+ //navigator
+ var footer = new List();
+ BuildHideRowsScript(uniqueId);
+
+ var paginateScriptFirst = BuildHideRowsScript(uniqueId) + GotoPageIndex(uniqueId, 0) + BuildPageScript(uniqueId, SIZE);
+ footer.Add(button[style: "margin: 2px;", onclick: paginateScriptFirst]("⏮"));
+
+ var paginateScriptPrevTen = BuildHideRowsScript(uniqueId) + UpdatePageIndex(uniqueId, -10, (maxRows - 1) / SIZE) + BuildPageScript(uniqueId, SIZE);
+ footer.Add(button[style: "margin: 2px;", onclick: paginateScriptPrevTen]("⏪"));
+
+ var paginateScriptPrev = BuildHideRowsScript(uniqueId) + UpdatePageIndex(uniqueId, -1, (maxRows - 1) / SIZE) + BuildPageScript(uniqueId, SIZE);
+ footer.Add(button[style: "margin: 2px;", onclick: paginateScriptPrev]("◀️"));
+
+ footer.Add(b[style: "margin: 2px;"]("Page"));
+ footer.Add(b[id: $"page_{uniqueId}", style: "margin: 2px;"]("1"));
+
+ var paginateScriptNext = BuildHideRowsScript(uniqueId) + UpdatePageIndex(uniqueId, 1, (maxRows - 1) / SIZE) + BuildPageScript(uniqueId, SIZE);
+ footer.Add(button[style: "margin: 2px;", onclick: paginateScriptNext]("▶️"));
+
+ var paginateScriptNextTen = BuildHideRowsScript(uniqueId) + UpdatePageIndex(uniqueId, 10, (maxRows - 1) / SIZE) + BuildPageScript(uniqueId, SIZE);
+ footer.Add(button[style: "margin: 2px;", onclick: paginateScriptNextTen]("⏩"));
+
+ var paginateScriptLast = BuildHideRowsScript(uniqueId) + GotoPageIndex(uniqueId, (maxRows - 1) / SIZE) + BuildPageScript(uniqueId, SIZE);
+ footer.Add(button[style: "margin: 2px;", onclick: paginateScriptLast]("⏭️"));
+
+ //table
+ var t = table[id: $"table_{uniqueId}"](
+ caption(title),
+ thead(tr(header)),
+ tbody(rows.Select(r => tr[style: "display: none"](r))),
+ tfoot(tr(td[colspan: df.Columns.Count + 1, style: "text-align: center;"](footer)))
+ );
+ writer.Write(t);
+
+ //show first page
+ writer.Write($"");
+ }
+ else
+ {
+ var rows = new List>();
+ for (var index = 0; index < df.Rows.Count; index++)
+ {
+ var cells = new List
+ {
+ td(i((index)))
+ };
+ foreach (var obj in df.Rows[index])
+ {
+ cells.Add(td(obj));
+ }
+ rows.Add(cells);
+ }
+
+ //table
+ var t = table[id: $"table_{uniqueId}"](
+ thead(tr(header)),
+ tbody(rows.Select(r => tr(r)))
+ );
+ writer.Write(t);
+ }
+ }, "text/html");
+ }
+
+ private static string BuildHideRowsScript(long uniqueId)
+ {
+ var script = $"var allRows = document.querySelectorAll('#table_{uniqueId} tbody tr:nth-child(n)'); ";
+ script += "for (let i = 0; i < allRows.length; i++) { allRows[i].style.display='none'; } ";
+ return script;
+ }
+
+ private static string BuildPageScript(long uniqueId, int size)
+ {
+ var script = $"var page = parseInt(document.querySelector('#page_{uniqueId}').innerHTML) - 1; ";
+ script += $"var pageRows = document.querySelectorAll(`#table_{uniqueId} tbody tr:nth-child(n + ${{page * {size} + 1 }})`); ";
+ script += $"for (let j = 0; j < {size}; j++) {{ pageRows[j].style.display='table-row'; }} ";
+ return script;
+ }
+
+ private static string GotoPageIndex(long uniqueId, long page)
+ {
+ var script = $"document.querySelector('#page_{uniqueId}').innerHTML = {page + 1}; ";
+ return script;
+ }
+
+ private static string UpdatePageIndex(long uniqueId, int step, long maxPage)
+ {
+ var script = $"var page = parseInt(document.querySelector('#page_{uniqueId}').innerHTML) - 1; ";
+ script += $"page = parseInt(page) + parseInt({step}); ";
+ script += $"page = page < 0 ? 0 : page; ";
+ script += $"page = page > {maxPage} ? {maxPage} : page; ";
+ script += $"document.querySelector('#page_{uniqueId}').innerHTML = page + 1; ";
+ return script;
+ }
+ }
+}
diff --git a/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj b/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj
new file mode 100644
index 0000000000..3167c983dc
--- /dev/null
+++ b/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj
@@ -0,0 +1,18 @@
+
+
+
+ netcoreapp3.1
+ false
+ $(NoWarn);MSML_ParameterLocalVarName;SA1028
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/Microsoft.Data.Analysis/ArrowStringDataFrameColumn.cs b/src/Microsoft.Data.Analysis/ArrowStringDataFrameColumn.cs
new file mode 100644
index 0000000000..a80e85c173
--- /dev/null
+++ b/src/Microsoft.Data.Analysis/ArrowStringDataFrameColumn.cs
@@ -0,0 +1,656 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Text;
+using Apache.Arrow;
+using Apache.Arrow.Types;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+
+namespace Microsoft.Data.Analysis
+{
+ ///
+ /// An immutable column to hold Arrow style strings
+ ///
+ public partial class ArrowStringDataFrameColumn : DataFrameColumn, IEnumerable
+ {
+ private IList> _dataBuffers;
+ private IList> _offsetsBuffers;
+ private IList> _nullBitMapBuffers;
+
+ ///
+ /// Constructs an empty with the given .
+ ///
+ /// The name of the column.
+ public ArrowStringDataFrameColumn(string name) : base(name, 0, typeof(string))
+ {
+ _dataBuffers = new List>();
+ _offsetsBuffers = new List>();
+ _nullBitMapBuffers = new List>();
+ }
+
+ ///
+ /// Constructs an with the given , and . The , and are the contents of the column in the Arrow format.
+ ///
+ /// The name of the column.
+ /// The Arrow formatted string values in this column.
+ /// The Arrow formatted offets in this column.
+ /// The Arrow formatted null bits in this column.
+ /// The length of the column.
+ /// The number of values in this column.
+ public ArrowStringDataFrameColumn(string name, ReadOnlyMemory values, ReadOnlyMemory offsets, ReadOnlyMemory nullBits, int length, int nullCount) : base(name, length, typeof(string))
+ {
+ ReadOnlyDataFrameBuffer dataBuffer = new ReadOnlyDataFrameBuffer(values, values.Length);
+ ReadOnlyDataFrameBuffer offsetBuffer = new ReadOnlyDataFrameBuffer(offsets, length + 1);
+ ReadOnlyDataFrameBuffer nullBitMapBuffer = new ReadOnlyDataFrameBuffer(nullBits, nullBits.Length);
+
+ if (length + 1 != offsetBuffer.Length)
+ throw new ArgumentException(nameof(offsetBuffer));
+
+ _dataBuffers = new List>();
+ _offsetsBuffers = new List>();
+ _nullBitMapBuffers = new List>();
+
+ _dataBuffers.Add(dataBuffer);
+ _offsetsBuffers.Add(offsetBuffer);
+ _nullBitMapBuffers.Add(nullBitMapBuffer);
+
+ _nullCount = nullCount;
+
+ }
+
+ private long _nullCount;
+
+ ///
+ public override long NullCount => _nullCount;
+
+ ///
+ /// Indicates if the value at this is .
+ ///
+ /// The index to look up.
+ /// A boolean value indicating the validity at this .
+ public bool IsValid(long index) => NullCount == 0 || GetValidityBit(index);
+
+ private bool GetValidityBit(long index)
+ {
+ if ((ulong)index > (ulong)Length)
+ {
+ throw new ArgumentOutOfRangeException(nameof(index));
+ }
+ // First find the right bitMapBuffer
+ int bitMapIndex = GetBufferIndexContainingRowIndex(index, out int indexInBuffer);
+ Debug.Assert(_nullBitMapBuffers.Count > bitMapIndex);
+ ReadOnlyDataFrameBuffer bitMapBuffer = _nullBitMapBuffers[bitMapIndex];
+ int bitMapBufferIndex = (int)((uint)index / 8);
+ Debug.Assert(bitMapBuffer.Length > bitMapBufferIndex);
+ byte curBitMap = bitMapBuffer[bitMapBufferIndex];
+ return ((curBitMap >> (indexInBuffer & 7)) & 1) != 0;
+ }
+
+ private void SetValidityBit(long index, bool value)
+ {
+ if ((ulong)index > (ulong)Length)
+ {
+ throw new ArgumentOutOfRangeException(nameof(index));
+ }
+ // First find the right bitMapBuffer
+ int bitMapIndex = GetBufferIndexContainingRowIndex(index, out int indexInBuffer);
+ Debug.Assert(_nullBitMapBuffers.Count > bitMapIndex);
+ DataFrameBuffer bitMapBuffer = (DataFrameBuffer)_nullBitMapBuffers[bitMapIndex];
+
+ // Set the bit
+ int bitMapBufferIndex = (int)((uint)indexInBuffer / 8);
+ Debug.Assert(bitMapBuffer.Length >= bitMapBufferIndex);
+ if (bitMapBuffer.Length == bitMapBufferIndex)
+ bitMapBuffer.Append(0);
+ byte curBitMap = bitMapBuffer[bitMapBufferIndex];
+ byte newBitMap;
+ if (value)
+ {
+ newBitMap = (byte)(curBitMap | (byte)(1 << (indexInBuffer & 7))); //bit hack for index % 8
+ if (((curBitMap >> (indexInBuffer & 7)) & 1) == 0 && indexInBuffer < Length - 1 && NullCount > 0)
+ {
+ // Old value was null.
+ _nullCount--;
+ }
+ }
+ else
+ {
+ if (((curBitMap >> (indexInBuffer & 7)) & 1) == 1 && indexInBuffer < Length)
+ {
+ // old value was NOT null and new value is null
+ _nullCount++;
+ }
+ else if (indexInBuffer == Length - 1)
+ {
+ // New entry from an append
+ _nullCount++;
+ }
+ newBitMap = (byte)(curBitMap & (byte)~(1 << (int)((uint)indexInBuffer & 7)));
+ }
+ bitMapBuffer[bitMapBufferIndex] = newBitMap;
+ }
+
+ ///
+ /// Returns an enumeration of immutable buffers representing the underlying values in the Apache Arrow format
+ ///
+ /// values are encoded in the buffers returned by GetReadOnlyNullBitmapBuffers in the Apache Arrow format
+ /// The offsets buffers returned by GetReadOnlyOffsetBuffers can be used to delineate each value
+ /// An enumeration of whose elements are the raw data buffers for the UTF8 string values.
+ public IEnumerable> GetReadOnlyDataBuffers()
+ {
+ for (int i = 0; i < _dataBuffers.Count; i++)
+ {
+ ReadOnlyDataFrameBuffer buffer = _dataBuffers[i];
+ yield return buffer.RawReadOnlyMemory;
+ }
+ }
+
+ ///
+ /// Returns an enumeration of immutable buffers representing values in the Apache Arrow format
+ ///
+ /// Each encodes the indices of values in its corresponding Data buffer
+ /// An enumeration of objects whose elements encode the null bit maps for the column's values
+ public IEnumerable> GetReadOnlyNullBitMapBuffers()
+ {
+ for (int i = 0; i < _nullBitMapBuffers.Count; i++)
+ {
+ ReadOnlyDataFrameBuffer buffer = _nullBitMapBuffers[i];
+ yield return buffer.RawReadOnlyMemory;
+ }
+ }
+
+ ///
+ /// Returns an enumeration of immutable representing offsets into its corresponding Data buffer.
+ /// The Apache Arrow format specifies how the offset buffer encodes the length of each value in the Data buffer
+ ///
+ /// An enumeration of objects.
+ public IEnumerable> GetReadOnlyOffsetsBuffers()
+ {
+ for (int i = 0; i < _offsetsBuffers.Count; i++)
+ {
+ ReadOnlyDataFrameBuffer buffer = _offsetsBuffers[i];
+ yield return buffer.ReadOnlyMemory;
+ }
+ }
+
+ // This is an immutable column, however this method exists to support Clone(). Keep this method private
+ // Appending a default string is equivalent to appending null. It increases the NullCount and sets a null bitmap bit
+ // Appending an empty string is valid. It does NOT affect the NullCount. It instead adds a new offset entry
+ private void Append(ReadOnlySpan value)
+ {
+ if (_dataBuffers.Count == 0)
+ {
+ _dataBuffers.Add(new DataFrameBuffer());
+ _nullBitMapBuffers.Add(new DataFrameBuffer());
+ _offsetsBuffers.Add(new DataFrameBuffer());
+ }
+ DataFrameBuffer mutableOffsetsBuffer = (DataFrameBuffer)_offsetsBuffers[_offsetsBuffers.Count - 1];
+ if (mutableOffsetsBuffer.Length == 0)
+ {
+ mutableOffsetsBuffer.Append(0);
+ }
+ Length++;
+ if (value == default)
+ {
+ mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1]);
+ }
+ else
+ {
+ DataFrameBuffer mutableDataBuffer = (DataFrameBuffer)_dataBuffers[_dataBuffers.Count - 1];
+ if (mutableDataBuffer.Length == ReadOnlyDataFrameBuffer.MaxCapacity)
+ {
+ mutableDataBuffer = new DataFrameBuffer();
+ _dataBuffers.Add(mutableDataBuffer);
+ _nullBitMapBuffers.Add(new DataFrameBuffer());
+ mutableOffsetsBuffer = new DataFrameBuffer();
+ _offsetsBuffers.Add(mutableOffsetsBuffer);
+ mutableOffsetsBuffer.Append(0);
+ }
+ mutableDataBuffer.EnsureCapacity(value.Length);
+ value.CopyTo(mutableDataBuffer.RawSpan.Slice(mutableDataBuffer.Length));
+ mutableDataBuffer.Length += value.Length;
+ mutableOffsetsBuffer.Append(mutableOffsetsBuffer[mutableOffsetsBuffer.Length - 1] + value.Length);
+ }
+ SetValidityBit(Length - 1, value != default);
+
+ }
+
+ private int GetBufferIndexContainingRowIndex(long rowIndex, out int indexInBuffer)
+ {
+ if (rowIndex >= Length)
+ {
+ throw new ArgumentOutOfRangeException(Strings.ColumnIndexOutOfRange, nameof(rowIndex));
+ }
+
+ // Since the strings here could be of variable length, scan linearly
+ int curArrayIndex = 0;
+ int numBuffers = _offsetsBuffers.Count;
+ while (curArrayIndex < numBuffers && rowIndex > _offsetsBuffers[curArrayIndex].Length - 1)
+ {
+ rowIndex -= _offsetsBuffers[curArrayIndex].Length - 1;
+ curArrayIndex++;
+ }
+ indexInBuffer = (int)rowIndex;
+ return curArrayIndex;
+ }
+
+ private ReadOnlySpan GetBytes(long index)
+ {
+ int offsetsBufferIndex = GetBufferIndexContainingRowIndex(index, out int indexInBuffer);
+ ReadOnlySpan offsetBufferSpan = _offsetsBuffers[offsetsBufferIndex].ReadOnlySpan;
+ int currentOffset = offsetBufferSpan[indexInBuffer];
+ int nextOffset = offsetBufferSpan[indexInBuffer + 1];
+ int numberOfBytes = nextOffset - currentOffset;
+ return _dataBuffers[offsetsBufferIndex].ReadOnlySpan.Slice(currentOffset, numberOfBytes);
+ }
+
+ ///
+ protected override object GetValue(long rowIndex) => GetValueImplementation(rowIndex);
+
+ private string GetValueImplementation(long rowIndex)
+ {
+ if (!IsValid(rowIndex))
+ {
+ return null;
+ }
+ var bytes = GetBytes(rowIndex);
+ unsafe
+ {
+ fixed (byte* data = &MemoryMarshal.GetReference(bytes))
+ return Encoding.UTF8.GetString(data, bytes.Length);
+ }
+ }
+
+ ///
+ protected override IReadOnlyList