nrnpyenv.sh determines PYTHONHOME more robustly. (#589)

* nrnpyenv.sh determines PYTHONHOME more robustly.

Uses sys.base_prefix whenever possible

Print the provenance of NRN_PYLIB.

On Linux, try to figure out the NRN_PYLIB using sysconfig LIBDIR but
search only in LIBDIR.
Additional method using h.libpython_path()

* More meaningful bash script names.

bin/nrnpyenv.sh

@@ -5,7 +5,7 @@
 # environment as python
 
 # May specify the python executable with explicit first argument.
-# Without arg use python and if that does not exist then python3
+# Without arg use python3 and if that does not exist then python
 
 # Overcome environment issues when --with-nrnpython=dynamic .
 
@@ -112,40 +112,60 @@ fi
 echo "# PYTHON=`$WHICH $PYTHON`"
 
 # what is the python library for Darwin
-z=''
+nrnpylib_provenance=""
+nrn_pylib=""
+kernel_name=''
 if type -P uname > /dev/null ; then
-  z=`uname`
+  kernel_name=`uname`
 fi
-if test "$z" = "Darwin" ; then
-  p=`$WHICH $PYTHON`
-  d=`dirname $p`
+if test "$kernel_name" = "Darwin" ; then
+  python_path=`$WHICH $PYTHON`
+  pyexedir=`dirname $python_path`
   # Get the python lib dir in an official way, working with virtualenv
-  PYLIB=$($p -c 'from distutils import sysconfig; print(sysconfig.get_config_var("LIBDIR"))')
+  PYLIB=$($python_path -c 'from distutils import sysconfig; print(sysconfig.get_config_var("LIBDIR"))')
   for path in $PYLIB/libpython*.dylib; do
     if test -f "$path"; then
-      l="$path"
+      nrn_pylib="$path"
       break
     fi
   done
-  if test -f "$l" ; then
-    z="$l"
-    unset p
-    unset d
-    unset l
-  else
+  if test -f "$nrn_pylib" ; then
+    unset python_path
+    unset pyexedir
+    nrnpylib_provenance="sysconfig LIBDIR"
+  fi
+  if test "$nrn_pylib" = "" ; then
+    nrn_pylib=$($p -c '
+try:
+  from neuron import h
+  shlib=h.libpython_path()
+  shlib = shlib if ".dylib" in shlib else ""
+  print(shlib)
+except:
+  print("")
+')
+    if test "$nrn_pylib" != "" ; then
+      nrnpylib_provenance="h.libpython_path()"
+    fi
+  fi
+  if test "$nrn_pylib" = "" ; then
     DYLD_PRINT_LIBRARIES=1
     export DYLD_PRINT_LIBRARIES
-    z=`$PYTHON -c 'quit()' 2>&1 | sed -n 's/^dyld: loaded: //p' | sed -n /libpython/p`
-    if test "$z" = "" ; then
-      z=`$PYTHON -c 'quit()' 2>&1 | sed -n 's/^dyld: loaded: //p' | sed -n 2p`
+    nrn_pylib=`$PYTHON -c 'quit()' 2>&1 | sed -n 's/^dyld: loaded: //p' | sed -n /libpython/p`
+    if test "$nrn_pylib" = "" ; then
+      nrn_pylib=`$PYTHON -c 'quit()' 2>&1 | sed -n 's/^dyld: loaded: //p' | sed -n 2p`
     fi
     unset DYLD_PRINT_LIBRARIES  
+    if test "$nrn_pylib" != "" ; then
+      nrnpylib_provenance=DYLD_PRINT_LIBRARIES
+    fi
   fi
-  if test -f "$z" ; then
-    PYLIB_DARWIN=$z
+  if test -f "$nrn_pylib" ; then
+    PYLIB_DARWIN=$nrn_pylib
   else
     PYLIB_DARWIN=""
   fi
+  export nrnpylib_provenance
   export PYLIB_DARWIN
 fi
 
@@ -157,6 +177,9 @@ import sys, os, site
 usep = "/"
 upathsep = ":"
 
+nrnpylib_provenance = "not found"
+nrnpyhome_provenance = "not found"
+
 def upath(path):
   #return linux path
   if path == None:
@@ -185,6 +208,7 @@ def u2d(p):
 
 #a copy of nrnpylib_linux() but with some os x specific modifications
 def nrnpylib_darwin_helper():
+  global nrnpylib_provenance
   import os, sys, re, subprocess
   #in case it was dynamically loaded by python
   pid = os.getpid()
@@ -203,6 +227,7 @@ def nrnpylib_darwin_helper():
       if re.search(r'libpython.*\.[ds]', line):
         print ("# nrn_pylib from lsof: %s" % line)
         nrn_pylib = line.strip()
+        nrnpylib_provenance = "lsof search for libpython..."
         return nrn_pylib
       if re.search(r'[Ll][Ii][Bb].*[Pp]ython', line):
         cnt += 1  
@@ -211,6 +236,7 @@ def nrnpylib_darwin_helper():
         if re.search(r'[Pp]ython', line.split('/')[-1]):
           print ("# nrn_pylib from lsof: %s" % line)
           nrn_pylib = line.strip()
+          nrnpylib_provenance = 'lsof search for second occurrence of [Ll][Ii][Bb].*[Pp]ython'
           return nrn_pylib
   else: # figure it out from the os path
     p = os.path.sep.join(os.__file__.split(os.path.sep)[:-1])
@@ -230,6 +256,7 @@ def nrnpylib_darwin_helper():
         libs.append(line.strip())
     print ('# %s'%str(libs))
     if len(libs) == 1:
+      nrnpylib_provenance="search based on os.__file__, found unique"
       print ("# nrn_pylib from os.path %s"%str(libs[0]))
       return libs[0]
     if len(libs) > 1:
@@ -256,19 +283,23 @@ def nrnpylib_darwin_helper():
       for i in libs:
         if name in i:
           print ("# nrn_pylib from os.path %s" % i)
+          nrnpylib_provenance='search based on os.__file__, found one with version in name'
           return i
       print ("# nrn_pylib from os.path %s" % str(nrn_pylib))
   return nrn_pylib
 
 def nrnpylib_darwin():
+  global nrnpylib_provenance
   import os
   nrn_pylib = os.getenv("PYLIB_DARWIN")
   if nrn_pylib != "":
     print ("# nrn_pylib from PYLIB_DARWIN %s"%nrn_pylib)
+    nrnpylib_provenance = os.getenv("nrnpylib_provenance")
     return nrn_pylib
   return nrnpylib_darwin_helper()
           
 def nrnpylib_mswin():
+  global nrnpylib_provenance
   import os, sys, re
   e = '/'.join(sys.executable.split(os.path.sep))
   cmd = 'cygcheck "%s"' % e
@@ -277,11 +308,36 @@ def nrnpylib_mswin():
   for line in f:
     if re.search('ython[a-zA-Z0-9_.]*\.dll', line):
       nrn_pylib = '/'.join(line.split(os.path.sep)).strip()
+      nrnpylib_provenance="cygcheck"
   return nrn_pylib
 
 def nrnpylib_linux():
+  global nrnpylib_provenance
   import os, sys, re, subprocess
+
+  # Try the official way first
+  from distutils import sysconfig
+  libdir=sysconfig.get_config_var("LIBDIR")
+  try:
+    from os.path import isfile, join
+    for f in os.listdir(libdir):
+      if 'libpython' in f and '.so' in f:
+        nrn_pylib = join(libdir, f)
+        nrnpylib_provenance='sysconfig LIBDIR'
+        return nrn_pylib
+  except:
+    pass
+  
   #in case it was dynamically loaded by python
+  try:
+    from neuron import h
+    s=h.libpython_path()
+    s = s if ".so" in s else ""
+    if (s != ""):
+      nrnpylib_provenance="h.libpython_path()"
+      return s
+  except:
+    print("")
   pid = os.getpid()
   cmd = "lsof -p %d"%pid
   f = []
@@ -297,6 +353,7 @@ def nrnpylib_linux():
       if re.search(r'libpython.*\.so', line):
         print ("# from lsof: %s" % line)
         nrn_pylib = line.strip()
+        nrnpylib_provenance = 'lsof search for libpython.*\.so'
         return nrn_pylib
   else: # figure it out from the os path
     p = os.path.sep.join(os.__file__.split(os.path.sep)[:-1])
@@ -316,6 +373,7 @@ def nrnpylib_linux():
         libs.append(line.strip())
     print ('# %s'%str(libs))
     if len(libs) == 1:
+      nrnpylib_provenance="search based on os.__file__, found unique"
       return libs[0]
     if len(libs) > 1:
       # which one do we want? Check the name of an imported shared object
@@ -335,6 +393,7 @@ def nrnpylib_linux():
           pass
       for i in libs:
         if name in i:
+          nrnpylib_provenance='search based on os.__file__, found one with version in name'
           return i
   return nrn_pylib
 
@@ -346,23 +405,32 @@ elif 'win' in sys.platform:
 elif 'linux' in sys.platform:
   nrn_pylib = nrnpylib_linux()
 
+#Use sys.base_prefix for PYTHONHOME if available, otherwise sys.prefix
+try:
+  sp = upath(sys.base_prefix)
+  spname='sys.base_prefix'
+  base=True
+except:
+  sp = upath(sys.prefix)
+  spname='sys.prefix'
+  base=False
+
 #there is a question about whether to use sys.prefix for PYTHONHOME
 #or whether to derive from site.__file__.
 #to help answer, ask how many sys.path items begin with sys.prefix and
 #how many begin with site.__file__ - 3
 p = [upath(i) for i in sys.path]
 print ("# items in sys.path = " + str(len(p)))
-sp = upath(sys.prefix)
 print ("# beginning with sys.prefix = " + str(len([i for i in p if sp in i])))
 s = usep.join(upath(site.__file__).split(usep)[:-3])
 if s == sp:
-  print ("# site-3 same as sys.prefix")
+  print ("# site-3 same as " + spname)
 else:
   print ("# beginning with site-3 = " + str(len([i for i in p if s in i])))
 foo = [i for i in p if sp not in i]
 foo = [i for i in foo if s not in i]
 print ("# in neither location " + str(foo))
-print ("# sys.prefix = " + sp)
+print ("# " + spname + " = " + sp)
 print ("# site-3 = " + s)
 	
 if "darwin" in sys.platform or "linux" in sys.platform or "win" in sys.platform:
@@ -374,7 +442,8 @@ if "darwin" in sys.platform or "linux" in sys.platform or "win" in sys.platform:
   if i > 1:
     path = newpath[:i]
 
-  pythonhome = upath(sys.prefix)
+  pythonhome = upath(sp)
+  print ("#pythonhome=" + pythonhome)
   pythonpath = upath(os.getenv("PYTHONPATH"))
 
   ldpath = ""
@@ -409,14 +478,18 @@ if "darwin" in sys.platform or "linux" in sys.platform or "win" in sys.platform:
   if pythonpath:
     print ("\n# if launch python, then need:")
     print ("export PYTHONPATH=" + dq + pythonpath + dq)
+
+  if path:
+    print ("\n#PYTHON prepended the following to PATH")
+    print ("export PATH=" + dq + path + "$PATH" + dq)
+
+  print("\n#NRN_PYLIB provenance: " + str(nrnpylib_provenance))
   print ("\n# if launch nrniv, then likely need:")
   if pythonhome:
     pythonhome=u2d(pythonhome)
     print ("export PYTHONHOME=" + dq + pythonhome + dq)
   if ldpath and nrn_pylib == None:
     print ("export LD_LIBRARY_PATH=" + dq + ldpath + upathsep + "$LD_LIBRARY_PATH" + dq)
-  if path:
-    print ("export PATH=" + dq + path + "$PATH" + dq)
   if nrn_pylib != None:
     print ('export NRN_PYLIB="%s"' % nrn_pylib)
 

src/nrnpython/nrnpy_hoc.cpp

@@ -9,6 +9,9 @@
 #include "nrnpy_utils.h"
 #include "../nrniv/shapeplt.h"
 #include <vector>
+#if defined(HAVE_DLFCN_H)
+#include <dlfcn.h>
+#endif
 
 #if defined(NRNPYTHON_DYNAMICLOAD) && NRNPYTHON_DYNAMICLOAD > 0
 // when compiled with different Python.h, force correct value
@@ -2671,6 +2674,25 @@ static PyObject* hocpickle_setstate(PyObject* self, PyObject* args) {
   return Py_None;
 }
 
+static PyObject* libpython_path(PyObject* self, PyObject* args) {
+#if defined(HAVE_DLFCN_H)
+  Dl_info info;
+  int rval = dladdr((const void*)Py_Initialize, &info);
+  if (!rval) {
+    PyErr_SetString(PyExc_Exception, "dladdr: Py_Initialize could not be matched to a shared object");
+    return NULL;
+  }
+  if (!info.dli_fname) {
+    PyErr_SetString(PyExc_Exception, "dladdr: No symbol matching Py_Initialize could be found.");
+    return NULL;
+  }
+  return Py_BuildValue("s", info.dli_fname);
+#else
+  Py_INCREF(Py_None);
+  return Py_None;
+#endif
+}
+
 // available for every HocObject
 static PyMethodDef hocobj_methods[] = {
     {"baseattr", hocobj_baseattr, METH_VARARGS,
@@ -2695,6 +2717,8 @@ static PyMethodDef toplevel_methods[] = {
      "Return a new Section"},
     {"setpointer", setpointer, METH_VARARGS,
      "Assign hoc variable address to NMODL POINTER"},
+    {"libpython_path", libpython_path, METH_NOARGS,
+     "Return full path to file that contains Py_Initialize()"},
     {NULL, NULL, 0, NULL}};
 
 static void add2topdict(PyObject* dict) {