Get Microsoft.Spark and Microsoft.Spark.Worker assembly version information

src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/SparkSessionExtensionsTests.cs

@@ -0,0 +1,41 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Linq;
+using Microsoft.Spark.Experimental.Sql;
+using Microsoft.Spark.Sql;
+using Xunit;
+
+namespace Microsoft.Spark.E2ETest.IpcTests
+{
+    [Collection("Spark E2E Tests")]
+    public class SparkSessionExtensionsTests
+    {
+        private readonly SparkSession _spark;
+
+        public SparkSessionExtensionsTests(SparkFixture fixture)
+        {
+            _spark = fixture.Spark;
+        }
+
+        [Fact]
+        public void TestVersion()
+        {
+            DataFrame versionDf = _spark.GetAssemblyInfo();
+            Row[] versionRows = versionDf.Collect().ToArray();
+            Assert.Equal(2, versionRows.Length);
+
+            Assert.Equal(
+                new string[] { "Microsoft.Spark", "Microsoft.Spark.Worker" },
+                versionRows.Select(r => r.GetAs<string>("AssemblyName")));
+            for (int i = 0; i < 2; ++i)
+            {
+                Assert.False(
+                    string.IsNullOrWhiteSpace(versionRows[i].GetAs<string>("AssemblyVersion")));
+                Assert.False(
+                    string.IsNullOrWhiteSpace(versionRows[i].GetAs<string>("HostName")));
+            }
+        }
+    }
+}

src/csharp/Microsoft.Spark/Experimental/Sql/SparkSessionExtensions.cs

@@ -0,0 +1,70 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Linq;
+using Microsoft.Spark.Sql;
+using Microsoft.Spark.Sql.Types;
+using static Microsoft.Spark.Experimental.Utils.AssemblyInfoProvider;
+using static Microsoft.Spark.Sql.Functions;
+
+namespace Microsoft.Spark.Experimental.Sql
+{
+    public static class SparkSessionExtensions
+    {
+        /// <summary>
+        /// Get the <see cref="AssemblyInfo"/> for the "Microsoft.Spark" assembly running
+        /// on the Spark Driver and make a "best effort" attempt in determining the
+        /// <see cref="AssemblyInfo"/> of "Microsoft.Spark.Worker"
+        /// assembly on the Spark Executors.
+        /// 
+        /// There is no guarantee that a Spark Executor will be run on all the nodes in
+        /// a cluster. To increase the likelyhood, the spark conf `spark.executor.instances`
+        /// and the <paramref name="numPartitions"/> settings should be adjusted to a
+        /// reasonable number relative to the number of nodes in the Spark cluster.
+        /// </summary>
+        /// <param name="session">The <see cref="SparkSession"/></param>
+        /// <param name="numPartitions">Number of partitions</param>
+        /// <returns>
+        /// A <see cref="DataFrame"/> containing the <see cref="AssemblyInfo"/>
+        /// </returns>
+        public static DataFrame GetAssemblyInfo(this SparkSession session, int numPartitions = 10)
+        {
+            var schema = new StructType(new StructField[]
+            {
+                new StructField("AssemblyName", new StringType(), isNullable: false),
+                new StructField("AssemblyVersion", new StringType(), isNullable: false),
+                new StructField("HostName", new StringType(), isNullable: false)
+            });
+
+            DataFrame driverAssmeblyInfoDf = session.CreateDataFrame(
+                new GenericRow[] { CreateGenericRow(MicrosoftSparkAssemblyInfo()) },
+                schema);
+
+            Func<Column, Column> executorAssemblyInfoUdf = Udf<int>(
+                i => CreateGenericRow(MicrosoftSparkWorkerAssemblyInfo()),
+                schema);
+            DataFrame df = session.CreateDataFrame(Enumerable.Range(0, 10 * numPartitions));
+
+            string tempColName = "ExecutorAssemblyInfo";
+            DataFrame executorAssemblyInfoDf = df
+                .Repartition(numPartitions)
+                .WithColumn(tempColName, executorAssemblyInfoUdf(df["_1"]))
+                .Select(schema.Fields.Select(f => Col($"{tempColName}.{f.Name}")).ToArray());
+
+            return driverAssmeblyInfoDf
+                .Union(executorAssemblyInfoDf)
+                .DropDuplicates()
+                .Sort(schema.Fields.Select(f => Col(f.Name)).ToArray());
+        }
+
+        private static GenericRow CreateGenericRow(AssemblyInfo assemblyInfo) =>
+            new GenericRow(new object[]
+            {
+                assemblyInfo.AssemblyName,
+                assemblyInfo.AssemblyVersion,
+                assemblyInfo.HostName
+            });
+    }
+}

src/csharp/Microsoft.Spark/Experimental/Utils/AssemblyInfoProvider.cs

@@ -0,0 +1,56 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Linq;
+using System.Net;
+using System.Reflection;
+
+namespace Microsoft.Spark.Experimental.Utils
+{
+    /// <summary>
+    /// Gets the <see cref="AssemblyInfo"/> for the "Microsoft.Spark" and "Microsoft.Spark.Worker"
+    /// assemblies if they exist within the current execution context of this application domain.
+    /// </summary>
+    internal static class AssemblyInfoProvider
+    {
+        private const string MicrosoftSparkAssemblyName = "Microsoft.Spark";
+        private const string MicrosoftSparkWorkerAssemblyName = "Microsoft.Spark.Worker";
+
+        private static readonly Lazy<AssemblyInfo> s_microsoftSparkAssemblyInfo =
+            new Lazy<AssemblyInfo>(() => CreateAssemblyInfo(MicrosoftSparkAssemblyName));
+
+        private static readonly Lazy<AssemblyInfo> s_microsoftSparkWorkerAssemblyInfo =
+            new Lazy<AssemblyInfo>(() => CreateAssemblyInfo(MicrosoftSparkWorkerAssemblyName));
+
+        internal static AssemblyInfo MicrosoftSparkAssemblyInfo() =>
+            s_microsoftSparkAssemblyInfo.Value;
+
+        internal static AssemblyInfo MicrosoftSparkWorkerAssemblyInfo() =>
+            s_microsoftSparkWorkerAssemblyInfo.Value;
+
+        private static AssemblyInfo CreateAssemblyInfo(string assemblyName)
+        {
+            Assembly assembly = AppDomain
+                .CurrentDomain
+                .GetAssemblies()
+                .Single(asm => asm.GetName().Name == assemblyName);
+
+            AssemblyName asmName = assembly.GetName();
+            return new AssemblyInfo
+            {
+                AssemblyName = asmName.Name,
+                AssemblyVersion = asmName.Version.ToString(),
+                HostName = Dns.GetHostName()
+            };
+        }
+
+        internal class AssemblyInfo
+        {
+            internal string AssemblyName { get; set; }
+            internal string AssemblyVersion { get; set; }
+            internal string HostName { get; set; }
+        }
+    }
+}