Update version check logic on the worker

src/csharp/Microsoft.Spark.Worker.UnitTest/TestData.cs

@@ -46,7 +46,7 @@ internal static Payload GetDefaultPayload()
             return new Payload()
             {
                 SplitIndex = 10,
-                Version = Versions.CurrentVersion,
+                Version = AssemblyInfoProvider.MicrosoftSparkAssemblyInfo().AssemblyVersion,
                 TaskContext = taskContext,
                 SparkFilesDir = "directory",
                 IncludeItems = new[] { "file1", "file2" },

src/csharp/Microsoft.Spark.Worker/TaskRunner.cs

@@ -13,6 +13,7 @@
 using Microsoft.Spark.Worker.Command;
 using Microsoft.Spark.Worker.Processor;
 using Microsoft.Spark.Worker.Utils;
+using static Microsoft.Spark.Utils.AssemblyInfoProvider;
 
 namespace Microsoft.Spark.Worker
 {
@@ -213,18 +214,15 @@ private Payload ProcessStream(
             }
         }
 
-        private void ValidateVersion(string versionStr)
+        private void ValidateVersion(string driverMicrosoftSparkVersion)
         {
-            // Initial version was shipped with version "1.0", so this needs to be adjusted
-            // to be compatible going forward.
-            if (versionStr == "1.0")
+            string workerVersion = MicrosoftSparkWorkerAssemblyInfo().AssemblyVersion;
+            // Worker is compatibile only within the same major version.
+            if (new Version(driverMicrosoftSparkVersion).Major != new Version(workerVersion).Major)
             {
-                versionStr = "0.1.0";
-            }
-
-            if (new Version(versionStr) < new Version(Versions.CurrentVersion))
-            {
-                throw new Exception($"Upgrade Microsoft.Spark to '{Versions.CurrentVersion}+' from '{versionStr}+'.");
+                throw new Exception("The major version of Microsoft.Spark.Worker " +
+                    $"({workerVersion}) does not match with Microsoft.Spark " +
+                    $"({driverMicrosoftSparkVersion}) on the driver.");
             }
         }
 

src/csharp/Microsoft.Spark/Experimental/Sql/SparkSessionExtensions.cs

@@ -6,7 +6,7 @@
 using System.Linq;
 using Microsoft.Spark.Sql;
 using Microsoft.Spark.Sql.Types;
-using static Microsoft.Spark.Experimental.Utils.AssemblyInfoProvider;
+using static Microsoft.Spark.Utils.AssemblyInfoProvider;
 using static Microsoft.Spark.Sql.Functions;
 
 namespace Microsoft.Spark.Experimental.Sql

src/csharp/Microsoft.Spark/Experimental/Utils/AssemblyInfoProvider.cs → src/csharp/Microsoft.Spark/Utils/AssemblyInfoProvider.cs

@@ -7,11 +7,12 @@
 using System.Net;
 using System.Reflection;
 
-namespace Microsoft.Spark.Experimental.Utils
+namespace Microsoft.Spark.Utils
 {
     /// <summary>
-    /// Gets the <see cref="AssemblyInfo"/> for the "Microsoft.Spark" and "Microsoft.Spark.Worker"
-    /// assemblies if they exist within the current execution context of this application domain.
+    /// Provides the <see cref="AssemblyInfo"/> for the "Microsoft.Spark" and
+    /// "Microsoft.Spark.Worker" assemblies if they exist within the current execution
+    /// context of this application domain.
     /// </summary>
     internal static class AssemblyInfoProvider
     {

src/csharp/Microsoft.Spark/Utils/UdfUtils.cs

@@ -178,7 +178,8 @@ internal static JvmObjectReference CreatePythonFunction(IJvmBridge jvm, byte[] c
                 CreateEnvVarsForPythonFunction(jvm),
                 arrayList, // Python includes
                 SparkEnvironment.ConfigurationService.GetWorkerExePath(),
-                Versions.CurrentVersion,
+                // Used to check the compatibility of UDFs between the driver and worker.
+                AssemblyInfoProvider.MicrosoftSparkAssemblyInfo().AssemblyVersion,
                 broadcastVariables,
                 null); // Accumulator
         }

src/csharp/Microsoft.Spark/Versions.cs

@@ -13,10 +13,5 @@ internal static class Versions
         internal const string V2_4_0 = "2.4.0";
         internal const string V2_4_2 = "2.4.2";
         internal const string V3_0_0 = "3.0.0";
-
-        // The following is used to check the compatibility of UDFs between
-        // the driver and worker side. This needs to be updated only when there
-        // is a breaking change on the UDF contract.
-        internal const string CurrentVersion = "0.9.0";
     }
 }