From 72d052e126c64bfbd58839688f3168cedff07a6e Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Tue, 23 Apr 2024 11:51:07 -0300 Subject: [PATCH] Release GIL when accessing Properties & Fields as we do for methods (#91) * Fix deadlock accessing properties/fields * Version bump to 2.0.36 --- src/embed_tests/Inheritance.cs | 23 +++++++++++-------- src/embed_tests/QCTest.cs | 12 +++++++--- src/embed_tests/TestUtil.cs | 6 +++++ src/perf_tests/Python.PerformanceTests.csproj | 4 ++-- src/runtime/Properties/AssemblyInfo.cs | 4 ++-- src/runtime/Py.cs | 9 ++++++++ src/runtime/Python.Runtime.csproj | 2 +- src/runtime/Types/FieldObject.cs | 23 +++++++++++++++---- src/runtime/Types/PropertyObject.cs | 11 +++++++-- 9 files changed, 70 insertions(+), 24 deletions(-) diff --git a/src/embed_tests/Inheritance.cs b/src/embed_tests/Inheritance.cs index ebbc24dc4..33bd659b5 100644 --- a/src/embed_tests/Inheritance.cs +++ b/src/embed_tests/Inheritance.cs @@ -182,18 +182,21 @@ public int XProp { get { - using (var scope = Py.CreateScope()) + using(Py.GIL()) { - scope.Set("this", this); - try + using (var scope = Py.CreateScope()) { - return scope.Eval($"super(this.__class__, this).{nameof(XProp)}"); - } - catch (PythonException ex) when (PythonReferenceComparer.Instance.Equals(ex.Type, Exceptions.AttributeError)) - { - if (this.extras.TryGetValue(nameof(this.XProp), out object value)) - return (int)value; - throw; + scope.Set("this", this); + try + { + return scope.Eval($"super(this.__class__, this).{nameof(XProp)}"); + } + catch (PythonException ex) when (PythonReferenceComparer.Instance.Equals(ex.Type, Exceptions.AttributeError)) + { + if (this.extras.TryGetValue(nameof(this.XProp), out object value)) + return (int)value; + throw; + } } } } diff --git a/src/embed_tests/QCTest.cs b/src/embed_tests/QCTest.cs index 5fd2afd29..ea90f96ab 100644 --- a/src/embed_tests/QCTest.cs +++ b/src/embed_tests/QCTest.cs @@ -102,8 +102,11 @@ public void TearDown() /// https://quantconnect.slack.com/archives/G51920EN4/p1615418516028900 public void ParamTest() { - var output = (bool)module.TestA(); - Assert.IsTrue(output); + using (Py.GIL()) + { + var output = (bool)module.TestA(); + Assert.IsTrue(output); + } } [TestCase("AAPL", false)] @@ -111,7 +114,10 @@ public void ParamTest() public void ContainsTest(string key, bool expected) { var dic = new Dictionary { { "SPY", new object() } }; - Assert.AreEqual(expected, (bool)containsTest(key, dic)); + using (Py.GIL()) + { + Assert.AreEqual(expected, (bool)containsTest(key, dic)); + } } [Test] diff --git a/src/embed_tests/TestUtil.cs b/src/embed_tests/TestUtil.cs index ab41d789c..c587473da 100644 --- a/src/embed_tests/TestUtil.cs +++ b/src/embed_tests/TestUtil.cs @@ -33,6 +33,12 @@ public class TestUtil [TestCase("PERatio10YearAverage", "pe_ratio_10_year_average")] [TestCase("CAPERatio", "cape_ratio")] [TestCase("EVToEBITDA3YearGrowth", "ev_to_ebitda_3_year_growth")] + [TestCase("EVToForwardEBITDA", "ev_to_forward_ebitda")] + [TestCase("EVToRevenue", "ev_to_revenue")] + [TestCase("EVToPreTaxIncome", "ev_to_pre_tax_income")] + [TestCase("EVToTotalAssets", "ev_to_total_assets")] + [TestCase("EVToFCF", "ev_to_fcf")] + [TestCase("EVToEBIT", "ev_to_ebit")] [TestCase("", "")] public void ConvertsNameToSnakeCase(string name, string expected) { diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index e80971f6f..1c427fd6f 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 20fea811f..0714dc6e2 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.35")] -[assembly: AssemblyFileVersion("2.0.35")] +[assembly: AssemblyVersion("2.0.36")] +[assembly: AssemblyFileVersion("2.0.36")] diff --git a/src/runtime/Py.cs b/src/runtime/Py.cs index 4f3fbf6d4..824cb9d15 100644 --- a/src/runtime/Py.cs +++ b/src/runtime/Py.cs @@ -10,12 +10,21 @@ namespace Python.Runtime; public static class Py { + public static IDisposable AllowThreads() => new AllowThreadsState(); public static GILState GIL() => PythonEngine.DebugGIL ? new DebugGILState() : new GILState(); public static PyModule CreateScope() => new(); public static PyModule CreateScope(string name) => new(name ?? throw new ArgumentNullException(nameof(name))); + public sealed class AllowThreadsState : IDisposable + { + private readonly IntPtr ts = PythonEngine.BeginAllowThreads(); + public void Dispose() + { + PythonEngine.EndAllowThreads(ts); + } + } public class GILState : IDisposable { diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index a3a5f1fdb..a83f17199 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.35 + 2.0.36 false LICENSE https://github.com/pythonnet/pythonnet diff --git a/src/runtime/Types/FieldObject.cs b/src/runtime/Types/FieldObject.cs index d33987f23..b8c7ed9c7 100644 --- a/src/runtime/Types/FieldObject.cs +++ b/src/runtime/Types/FieldObject.cs @@ -64,11 +64,18 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference // Fasterflect does not support constant fields if (info.IsLiteral && !info.IsInitOnly) { - result = info.GetValue(null); + using (Py.AllowThreads()) + { + result = info.GetValue(null); + } } else { - result = self.GetMemberGetter(info.DeclaringType)(info.DeclaringType); + var getter = self.GetMemberGetter(info.DeclaringType); + using (Py.AllowThreads()) + { + result = getter(info.DeclaringType); + } } return Converter.ToPython(result, info.FieldType); @@ -92,12 +99,20 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference // Fasterflect does not support constant fields if (info.IsLiteral && !info.IsInitOnly) { - result = info.GetValue(co.inst); + using (Py.AllowThreads()) + { + result = info.GetValue(co.inst); + } } else { var type = co.inst.GetType(); - result = self.GetMemberGetter(type)(self.IsValueType(type) ? co.inst.WrapIfValueType() : co.inst); + var getter = self.GetMemberGetter(type); + var argument = self.IsValueType(type) ? co.inst.WrapIfValueType() : co.inst; + using (Py.AllowThreads()) + { + result = getter(argument); + } } return Converter.ToPython(result, info.FieldType); diff --git a/src/runtime/Types/PropertyObject.cs b/src/runtime/Types/PropertyObject.cs index 557122958..a274e91e4 100644 --- a/src/runtime/Types/PropertyObject.cs +++ b/src/runtime/Types/PropertyObject.cs @@ -76,7 +76,11 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference try { - result = self.GetMemberGetter(info.DeclaringType)(info.DeclaringType); + var getterFunc = self.GetMemberGetter(info.DeclaringType); + using (Py.AllowThreads()) + { + result = getterFunc(info.DeclaringType); + } return Converter.ToPython(result, info.PropertyType); } catch (Exception e) @@ -93,7 +97,10 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference try { - result = getter.Invoke(co.inst, Array.Empty()); + using (Py.AllowThreads()) + { + result = getter.Invoke(co.inst, Array.Empty()); + } return Converter.ToPython(result, info.PropertyType); } catch (Exception e)