Improve installation of Python packages (#939)

Improve installation of Python packages and include the following in the distribution:

- utility programs like dlite-getuuid, dlite-codegen, patch-activate.sh
- storages, like blob.json
- README.md and LICENSE files

In addition it fixed a bug in the setup.py script that in addition to dlite-python also installs a pip package called dlite. If you remove that you will remove pip, hense messing with your entire environment.

---------

Co-authored-by: Francesca L. Bleken <48128015+francescalb@users.noreply.github.com>

CMakeLists.txt

@@ -770,6 +770,9 @@ add_custom_target(show ${cmd})
 # Subdirectories
 add_subdirectory(src)
 
+# Tools - may depend on storage plugins
+add_subdirectory(tools)
+
 # Storage plugins
 add_subdirectory(storages/json)
 if(WITH_HDF5)
@@ -783,9 +786,6 @@ if(WITH_PYTHON)
   add_subdirectory(storages/python)
 endif()
 
-# Tools - may depend on storage plugins
-add_subdirectory(tools)
-
 # Fortran - depends on tools
 if(WITH_FORTRAN)
   add_subdirectory(bindings/fortran)

bindings/python/CMakeLists.txt

@@ -15,8 +15,8 @@ set(py_sources
 
 # Python sub-packages
 set(py_packages
-  triplestore
-  )
+  #triplestore
+)
 
 configure_file(paths.py.in paths.py)
 if(dlite_PYTHON_BUILD_REDISTRIBUTABLE_PACKAGE)
@@ -187,6 +187,7 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-storage-plugins
   COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-mapping-plugins
   COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/storages
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/bin
   COMMAND ${CMAKE_COMMAND} -E copy_if_different
     "$<TARGET_FILE:dlite>"
     ${pkgdir}
@@ -199,14 +200,34 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E copy_if_different
     "$<TARGET_FILE:dlite-plugins-json>"
     ${pkgdir}/share/dlite/storage-plugins
-  COMMAND ${CMAKE_COMMAND} -E copy_directory
-    ${dlite_SOURCE_DIR}/storages/python/python-storage-plugins
-    ${pkgdir}/share/dlite/python-storage-plugins
-  COMMAND ${CMAKE_COMMAND} -E copy_directory
-    ${dlite_SOURCE_DIR}/bindings/python/python-mapping-plugins
-    ${pkgdir}/share/dlite/python-mapping-plugins
+  COMMAND ${CMAKE_COMMAND}
+    -DSOURCE_DIR=${dlite_SOURCE_DIR}/storages/python/python-storage-plugins
+    -DDEST_DIR=${pkgdir}/share/dlite/python-storage-plugins
+    -DPATTERN="*.py"
+    -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake
+  COMMAND ${CMAKE_COMMAND}
+    -DSOURCE_DIR=${dlite_SOURCE_DIR}/bindings/python/python-mapping-plugins
+    -DDEST_DIR=${pkgdir}/share/dlite/python-mapping-plugins
+    -DPATTERN="*.py"
+    -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake
+  COMMAND ${CMAKE_COMMAND}
+    -DSOURCE_DIR=${dlite_SOURCE_DIR}/storages/python/python-storage-plugins
+    -DDEST_DIR=${pkgdir}/share/dlite/storages
+    -DPATTERN="*.json"
+    -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${dlite_SOURCE_DIR}/README.md ${dlite_SOURCE_DIR}/LICENSE
+    ${pkgdir}/share/dlite
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    "$<TARGET_FILE:dlite-codegen>"
+    "$<TARGET_FILE:dlite-env>"
+    "$<TARGET_FILE:dlite-getuuid>"
+    ${pkgdir}/share/dlite/bin
   DEPENDS
     python_package
+    dlite-codegen
+    dlite-env
+    dlite-getuuid
   )
 
 #

bindings/python/dlite-path-python.i

@@ -43,4 +43,24 @@ mapping_plugin_path = FUPath("mapping-plugins")
 python_storage_plugin_path = FUPath("python-storage-plugins")
 python_mapping_plugin_path = FUPath("python-mapping-plugins")
 
+# Update default search paths
+from pathlib import Path
+pkgdir = Path(__file__).resolve().parent
+sharedir = pkgdir / "share" / "dlite"
+if (sharedir / "storages").exists():
+    storage_path[-1] = sharedir / "storages"
+    #storage_path.append(sharedir / "storages")
+if (sharedir / "storage-plugins").exists():
+    storage_plugin_path[-1] = sharedir / "storage-plugins"
+    #storage_plugin_path.append(sharedir / "storage-plugins")
+if (sharedir / "mapping-plugins").exists():
+    mapping_plugin_path[-1] = sharedir / "mapping-plugins"
+    #mapping_plugin_path.append(sharedir / "mapping-plugins")
+if (sharedir / "python-storage-plugins").exists():
+    python_storage_plugin_path[-1] = sharedir / "python-storage-plugins"
+    #python_storage_plugin_path.append(sharedir / "python-storage-plugins")
+if (sharedir / "python-mapping-plugins").exists():
+    python_mapping_plugin_path[-1] = sharedir / "python-mapping-plugins"
+    #python_mapping_plugin_path.append(sharedir / "python-mapping-plugins")
+
 %}

bindings/python/tests/global_dlite_state_mod2.py

@@ -1,4 +1,7 @@
-#!/usr/bin/env python
+# This file is executed by test_global_dlite_state.py
+#
+# Note that entitydir is defined in the global scope, so it should
+# not be redefined here
 from pathlib import Path
 
 import dlite
@@ -7,9 +10,6 @@
 assert len(dlite.istore_get_uuids()) == 3 + 3
 
 
-thisdir = Path(__file__).absolute().parent
-entitydir = thisdir / "entities"
-
 url = f"json://{entitydir}/MyEntity.json"
 
 # myentity is already defined via test_global_dlite_state, no new instance is added to istore

bindings/python/tests/test_transaction.py

@@ -7,7 +7,7 @@
 # Configure paths
 thisdir = Path(__file__).parent.absolute()
 
-dlite.storage_path.append(thisdir / "*.json")
+dlite.storage_path.append(thisdir / "entities" / "*.json")
 Person = dlite.get_instance("http://onto-ns.com/meta/0.1/Person")
 
 person = Person(dimensions={"N": 4})

cmake/CopyDirectory.cmake

@@ -0,0 +1,12 @@
+# Copy directory
+#
+# Parameters (passed with -D)
+#  - SOURCE_DIR: directory to copy
+#  - DEST_DIR: new destination directory
+#  - PATTERN: pattern matching files to include
+#
+file(
+  COPY "${SOURCE_DIR}/"
+  DESTINATION "${DEST_DIR}"
+  FILES_MATCHING PATTERN "${PATTERN}"
+)

doc/contributors_guide/tips_and_tricks.md

@@ -6,6 +6,39 @@ Setting up a virtual Python environment for building dlite
 See [Build against Python environment] in the installation instructions.
 
 
+Test installation before releasing on PyPI
+------------------------------------------
+If you have updated the installation of the [dlite-python] package, or you get failures on GitHub CI/CD that cannot be reproduced locally, you might want to test installing dlite-python before releasing it on PyPI.
+
+This can be done as follows:
+
+1. Create a new virtual environment and install requirements and the wheel package
+
+       python -m venv ~/.envs/testenv
+       source ~/.envs/testenv/bin/activate
+       pip install -U pip
+       pip install wheel -r requirements.txt
+
+2. Build the wheel
+
+       cd python
+       python setup.py bdist_wheel
+
+3. Install the wheel with pip in a newly created environment (the version numbers may differ for your case)
+
+       pip install dist/DLite_Python-0.5.22-cp311-cp311-linux_x86_64.whl
+
+4. Finally, test by importing dlite in the standard manner
+
+       cd ..
+       python
+       >>> import dlite
+
+   or you can run the python tests
+
+       python bindings/python/tests/test_python_bindings.py
+
+
 Debugging Python storage plugins
 --------------------------------
 Exceptions occurring inside Python storage plugins are not propagated to the calling interpreter, and will therefor not be shown.
@@ -192,6 +225,7 @@ More useful gdb commands:
 
 [virtualenvwrapper]: https://pypi.org/project/virtualenvwrapper/
 [Build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html#build-against-python-environment
+[dlite-python]: https://pypi.org/project/DLite-Python/
 [valgrind]: http://valgrind.org/
 [gdb]: https://sourceware.org/gdb/
 [GDB Tutorial]: https://www.gdbtutorial.com/

python/.gitignore

@@ -2,5 +2,7 @@
 
 dist/
 wheelhouse/
-dlite_python.egg-info
+build/
+dlite_python.egg-info/
+DLite_Python.egg-info/
 .eggs

python/setup.py

@@ -2,14 +2,16 @@
 import sys
 import platform
 import re
+import shutil
 import site
 import subprocess
-from shutil import copytree
+from glob import glob
 from typing import TYPE_CHECKING
 from pathlib import Path
 
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
+from setuptools.command.install import install
 
 if TYPE_CHECKING:
     from typing import Union
@@ -120,8 +122,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
         build_type = "Debug" if self.debug else "Release"
         cmake_args = [
             "cmake",
-            str(ext.sourcedir),
             f"-DCMAKE_CONFIGURATION_TYPES:STRING={build_type}",
+            str(ext.sourcedir),
         ]
         cmake_args.extend(CMAKE_ARGS)
         cmake_args.extend(environment_cmake_args)
@@ -134,7 +136,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
                 cwd=self.build_temp,
                 env=env,
                 capture_output=True,
-                check=True)
+                check=True,
+            )
         except subprocess.CalledProcessError as e:
             print("stdout:", e.stdout.decode("utf-8"), "\n\nstderr:",
                   e.stderr.decode("utf-8"))
@@ -145,24 +148,36 @@ def build_extension(self, ext: CMakeExtension) -> None:
                 cwd=self.build_temp,
                 env=env,
                 capture_output=True,
-                check=True
+                check=True,
             )
         except subprocess.CalledProcessError as e:
             print("stdout:", e.stdout.decode("utf-8"), "\n\nstderr:",
                   e.stderr.decode("utf-8"))
             raise
 
         cmake_bdist_dir = Path(self.build_temp) / Path(ext.python_package_dir)
-        copytree(
+        shutil.copytree(
             str(cmake_bdist_dir / ext.name),
             str(Path(output_dir) / ext.name),
             dirs_exist_ok=True,
         )
 
+
+class CustomInstall(install):
+    """Custom handler for the 'install' command."""
+    def run(self):
+        super().run()
+        bindir = Path(self.build_lib) / "dlite" / "share" / "dlite" / "bin"
+        # Possible to make a symlink instead of copy to save space
+        for prog in glob(str(bindir / "*")):
+            shutil.copy(prog, self.install_scripts)
+
+
 version = re.search(
     r"project\([^)]*VERSION\s+([0-9.]+)",
     (SOURCE_DIR / "CMakeLists.txt").read_text(),
 ).groups()[0]
+share = Path(".") / "share" / "dlite"
 
 setup(
     name="DLite-Python",
@@ -198,20 +213,20 @@ def build_extension(self, ext: CMakeExtension) -> None:
     install_requires="numpy>=1.14.5,<1.27.0",
     #install_requires=requirements,
     #extras_require=extra_requirements,
-    packages=["dlite"],
+    packages=["DLite-Python"],
     scripts=[
         str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"),
+        str(SOURCE_DIR / "cmake" / "patch-activate.sh"),
     ],
     package_data={
         "dlite": [
-            dlite_compiled_ext,
-            dlite_compiled_dll_suffix,
-            str(Path(".") / "share" / "dlite" / "storage-plugins" /
-                dlite_compiled_dll_suffix),
-            str(Path(".") / "bin" / "dlite-getuuid"),
-            str(Path(".") / "bin" / "dlite-codegen"),
-            str(Path(".") / "bin" / "dlite-env"),
-            str(Path(".") / "bin" / "patch-activate.sh"),
+            str(share / "README.md"),
+            str(share / "LICENSE"),
+            str(share / "storage-plugins" / dlite_compiled_dll_suffix),
+            str(share / "mapping-plugins" / dlite_compiled_dll_suffix),
+            str(share / "python-storage-plugins" / "*.py"),
+            str(share / "python-mapping-plugins" / "*.py"),
+            str(share / "storages" / "*.json"),
         ]
     },
     ext_modules=[
@@ -223,6 +238,7 @@ def build_extension(self, ext: CMakeExtension) -> None:
     ],
     cmdclass={
         "build_ext": CMakeBuildExt,
+        "install": CustomInstall,
     },
     zip_safe=False,
 )

src/tests/mappings/ent3.json

@@ -1,7 +1,5 @@
 {
-  "name": "ent3",
-  "version": "0.1",
-  "namespace": "http://onto-ns.com/meta",
+  "uri": "http://onto-ns.com/meta/0.1/ent3",
   "description": "test entity",
   "dimensions": [],
   "properties": [
@@ -11,5 +9,4 @@
       "unit": "hundreds"
     }
   ],
-  "dataname": "http://onto-ns.com/meta/0.1/ent1"
-}
+}

src/tests/python/test_python_mapping.c

@@ -42,12 +42,14 @@ MU_TEST(test_initialize)
 
 MU_TEST(test_map)
 {
-  DLiteInstance *insts[1], *inst3;
+  DLiteInstance *insts[1], *inst3, *ent3;
   const DLiteInstance **instances = (const DLiteInstance **)insts;
   void *p;
   instances[0] = dlite_instance_get("2daa6967-8ecd-4248-97b2-9ad6fefeac14");
   mu_check(instances[0]);
 
+  ent3 = dlite_instance_get("http://onto-ns.com/meta/0.1/ent3");
+  mu_check(ent3);
   inst3 = dlite_mapping("http://onto-ns.com/meta/0.1/ent3", instances, 1);
   mu_check(inst3);
   mu_check((p = dlite_instance_get_property(inst3, "c")));