FIX-#3774: Fix segfault at init if only omnisci present (#3783)

* always import omnisci with correct dlopen flags
* add a section on the issue in troubleshooting
* worked around incompatibility of OmniSci and pyarrow.gandiva

Signed-off-by: Igoshev, Yaroslav <yaroslav.igoshev@intel.com>

docs/developer/troubleshooting.rst

@@ -111,10 +111,10 @@ differntly. Example of such behaviour is shown below.
   try :
       with open(test_filename, "w") as f:
           f.write(data)
-      
+
       pandas_df = pandas.read_csv(test_filename, **kwargs)
       pd_df = pd.read_csv(test_filename, **kwargs)
-      
+
       print(pandas_df)
       print(pd_df)
   finally:
@@ -195,4 +195,25 @@ funcion for example) as it is shown in the example below:
   pd_df = pd_df.set_index(index_col_name)
   pd_df.index.name = None
 
+Error when using OmniSci engine along with ``pyarrow.gandiva``: ``LLVM ERROR: inconsistency in registered CommandLine options``
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+This can happen when you use OmniSci engine along with ``pyarrow.gandiva``:
+
+.. code-block:: python
+
+  import modin.config as cfg
+  cfg.Engine.put("Native")  # 'omniscidbe'/'dbe' would be imported with dlopen flags
+  cfg.StorageFormat.put("Omnisci")
+  cfg.IsExperimental.put(True)
+  import modin.pandas as pd
+  import pyarrow.gandiva as gandiva  # Error
+  CommandLine Error: Option 'enable-vfe' registered more than once!
+  LLVM ERROR: inconsistency in registered CommandLine options
+  Aborted (core dumped)
+
+**Solution**
+
+Do not use OmniSci engine along with ``pyarrow.gandiva``.
+
 .. _issue: https://github.com/modin-project/modin/issues

docs/developer/using_omnisci.rst

@@ -21,4 +21,8 @@ or use it in your code:
 Since OmniSci is run through its native engine Modin itself sets ``MODIN_ENGINE=Native``
 and you might not specify it explicitly.
 
+.. note::
+   If you encounter ``LLVM ERROR: inconsistency in registered CommandLine options`` error when using OmniSci,
+   please refer to the respective section in :doc:`Troubleshooting </developer/troubleshooting>` page to avoid the issue.
+
 .. _OmnisciDB: https://www.omnisci.com/platform/omniscidb

modin/config/envvars.py

@@ -109,14 +109,13 @@ def _get_default(cls):
                 )
             return "Dask"
         try:
-            from omniscidbe import PyDbEngine  # noqa
-        except ModuleNotFoundError:
-            try:
-                from dbe import PyDbEngine  # noqa
-            except ImportError:
-                pass
-            else:
-                return "Native"
+            # We import ``PyDbEngine`` from this module since correct import of ``PyDbEngine`` itself
+            # from Omnisci is located in it with all the necessary options for dlopen.
+            from modin.experimental.core.execution.native.implementations.omnisci_on_native.utils import (  # noqa
+                PyDbEngine,
+            )
+        except ImportError:
+            pass
         else:
             return "Native"
         raise ImportError(

modin/core/storage_formats/pyarrow/query_compiler.py

@@ -26,7 +26,6 @@
 from pandas.core.computation.ops import UnaryOp, BinOp, Term, MathCall, Constant
 
 import pyarrow as pa
-import pyarrow.gandiva as gandiva
 
 
 class FakeSeries:
@@ -221,6 +220,11 @@ def gandiva_query(table, query):
             expr = gen_table_expr(table, query)
             if not can_be_condition(expr):
                 raise ValueError("Root operation should be a filter.")
+
+            # We use this import here because of https://github.com/modin-project/modin/issues/3849,
+            # after the issue is fixed we should put the import at the top of this file
+            import pyarrow.gandiva as gandiva
+
             builder = gandiva.TreeExprBuilder()
             root = build_node(table, expr.terms, builder)
             cond = builder.make_condition(root)

modin/experimental/core/execution/native/implementations/omnisci_on_native/omnisci_worker.py

@@ -14,23 +14,12 @@
 """Module provides ``OmnisciServer`` class."""
 
 import uuid
-import sys
 import os
 
 import pyarrow as pa
 import numpy as np
 
-if sys.platform == "linux":
-    prev = sys.getdlopenflags()
-    sys.setdlopenflags(1 | 256)  # RTLD_LAZY+RTLD_GLOBAL
-
-try:
-    from omniscidbe import PyDbEngine
-except ModuleNotFoundError:  # fallback for older omniscidbe4py package naming
-    from dbe import PyDbEngine
-
-if sys.platform == "linux":
-    sys.setdlopenflags(prev)
+from .utils import PyDbEngine
 
 from modin.config import OmnisciFragmentSize, OmnisciLaunchParameters
 

modin/experimental/core/execution/native/implementations/omnisci_on_native/utils.py

@@ -0,0 +1,30 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+"""Utilities for OmniSci on Native execution."""
+
+import os
+import sys
+
+
+if sys.platform == "linux":
+    prev = sys.getdlopenflags()
+    sys.setdlopenflags(os.RTLD_LAZY | os.RTLD_GLOBAL)
+
+try:
+    from omniscidbe import PyDbEngine  # noqa
+except ModuleNotFoundError:  # fallback for older omniscidbe4py package naming
+    from dbe import PyDbEngine  # noqa
+finally:
+    if sys.platform == "linux":
+        sys.setdlopenflags(prev)

setup.cfg

@@ -27,7 +27,6 @@ per-file-ignores =
     stress_tests/kaggle/*:E402
     modin/experimental/pandas/__init__.py:E402
     modin/_version.py:T001
-    modin/experimental/core/execution/native/implementations/omnisci_on_native/omnisci_worker.py:E402
     modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py:E402
 
 [coverage:run]