From e43afcdd0a7bb80e5c61b3e983130a624ac4426e Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 3 Sep 2024 21:51:51 +0200 Subject: [PATCH 1/6] Updated installation of Python packages - include utility programs like dlite-getuuid - include storages - include README.md and LICENSE --- CMakeLists.txt | 6 +-- bindings/python/CMakeLists.txt | 37 ++++++++++++++---- cmake/CopyDirectory.cmake | 12 ++++++ python/.gitignore | 4 +- python/{dlite => DLite-Python}/.gitignore | 0 python/{dlite => DLite-Python}/__init__.py | 0 python/MANIFEST.in | 2 - python/setup.py | 44 +++++++++++++++------- 8 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 cmake/CopyDirectory.cmake rename python/{dlite => DLite-Python}/.gitignore (100%) rename python/{dlite => DLite-Python}/__init__.py (100%) delete mode 100644 python/MANIFEST.in diff --git a/CMakeLists.txt b/CMakeLists.txt index e25972506..26a2a83c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -770,6 +770,9 @@ add_custom_target(show ${cmd}) # Subdirectories add_subdirectory(src) +# Tools - may depend on storage plugins +add_subdirectory(tools) + # Storage plugins add_subdirectory(storages/json) if(WITH_HDF5) @@ -783,9 +786,6 @@ if(WITH_PYTHON) add_subdirectory(storages/python) endif() -# Tools - may depend on storage plugins -add_subdirectory(tools) - # Fortran - depends on tools if(WITH_FORTRAN) add_subdirectory(bindings/fortran) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 8a5f81312..e12a0c60e 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -15,8 +15,8 @@ set(py_sources # Python sub-packages set(py_packages - triplestore - ) + #triplestore +) configure_file(paths.py.in paths.py) if(dlite_PYTHON_BUILD_REDISTRIBUTABLE_PACKAGE) @@ -187,6 +187,7 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-storage-plugins COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/python-mapping-plugins COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/storages + COMMAND ${CMAKE_COMMAND} -E make_directory ${pkgdir}/share/dlite/bin COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" ${pkgdir} @@ -199,14 +200,34 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" ${pkgdir}/share/dlite/storage-plugins - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${dlite_SOURCE_DIR}/storages/python/python-storage-plugins - ${pkgdir}/share/dlite/python-storage-plugins - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${dlite_SOURCE_DIR}/bindings/python/python-mapping-plugins - ${pkgdir}/share/dlite/python-mapping-plugins + COMMAND ${CMAKE_COMMAND} + -DSOURCE_DIR=${dlite_SOURCE_DIR}/storages/python/python-storage-plugins + -DDEST_DIR=${pkgdir}/share/dlite/python-storage-plugins + -DPATTERN="*.py" + -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake + COMMAND ${CMAKE_COMMAND} + -DSOURCE_DIR=${dlite_SOURCE_DIR}/bindings/python/python-mapping-plugins + -DDEST_DIR=${pkgdir}/share/dlite/python-mapping-plugins + -DPATTERN="*.py" + -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake + COMMAND ${CMAKE_COMMAND} + -DSOURCE_DIR=${dlite_SOURCE_DIR}/storages/python/python-storage-plugins + -DDEST_DIR=${pkgdir}/share/dlite/storages + -DPATTERN="*.json" + -P ${dlite_SOURCE_DIR}/cmake/CopyDirectory.cmake + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${dlite_SOURCE_DIR}/README.md ${dlite_SOURCE_DIR}/LICENSE + ${pkgdir}/share/dlite + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_BINARY_DIR}/tools/dlite-codegen${EXEEXT} + ${CMAKE_BINARY_DIR}/tools/dlite-env${EXEEXT} + ${CMAKE_BINARY_DIR}/tools/dlite-getuuid${EXEEXT} + ${pkgdir}/share/dlite/bin DEPENDS python_package + dlite-codegen + dlite-env + dlite-getuuid ) # diff --git a/cmake/CopyDirectory.cmake b/cmake/CopyDirectory.cmake new file mode 100644 index 000000000..a1d72df52 --- /dev/null +++ b/cmake/CopyDirectory.cmake @@ -0,0 +1,12 @@ +# Copy directory +# +# Parameters (passed with -D) +# - SOURCE_DIR: directory to copy +# - DEST_DIR: new destination directory +# - PATTERN: pattern matching files to include +# +file( + COPY "${SOURCE_DIR}/" + DESTINATION "${DEST_DIR}" + FILES_MATCHING PATTERN "${PATTERN}" +) diff --git a/python/.gitignore b/python/.gitignore index 0b5510797..993ad7cc1 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -2,5 +2,7 @@ dist/ wheelhouse/ -dlite_python.egg-info +build/ +dlite_python.egg-info/ +DLite_Python.egg-info/ .eggs diff --git a/python/dlite/.gitignore b/python/DLite-Python/.gitignore similarity index 100% rename from python/dlite/.gitignore rename to python/DLite-Python/.gitignore diff --git a/python/dlite/__init__.py b/python/DLite-Python/__init__.py similarity index 100% rename from python/dlite/__init__.py rename to python/DLite-Python/__init__.py diff --git a/python/MANIFEST.in b/python/MANIFEST.in deleted file mode 100644 index 154c1eed7..000000000 --- a/python/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include ../LICENSE ../README.md ../CMakeLists.txt pyproject.toml -recursive-include ../. **/* diff --git a/python/setup.py b/python/setup.py index d3090719b..aa1402af0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,14 +2,16 @@ import sys import platform import re +import shutil import site import subprocess -from shutil import copytree +from glob import glob from typing import TYPE_CHECKING from pathlib import Path from setuptools import Extension, setup from setuptools.command.build_ext import build_ext +from setuptools.command.install import install if TYPE_CHECKING: from typing import Union @@ -120,8 +122,8 @@ def build_extension(self, ext: CMakeExtension) -> None: build_type = "Debug" if self.debug else "Release" cmake_args = [ "cmake", - str(ext.sourcedir), f"-DCMAKE_CONFIGURATION_TYPES:STRING={build_type}", + str(ext.sourcedir), ] cmake_args.extend(CMAKE_ARGS) cmake_args.extend(environment_cmake_args) @@ -134,7 +136,8 @@ def build_extension(self, ext: CMakeExtension) -> None: cwd=self.build_temp, env=env, capture_output=True, - check=True) + check=True, + ) except subprocess.CalledProcessError as e: print("stdout:", e.stdout.decode("utf-8"), "\n\nstderr:", e.stderr.decode("utf-8")) @@ -145,7 +148,7 @@ def build_extension(self, ext: CMakeExtension) -> None: cwd=self.build_temp, env=env, capture_output=True, - check=True + check=True, ) except subprocess.CalledProcessError as e: print("stdout:", e.stdout.decode("utf-8"), "\n\nstderr:", @@ -153,16 +156,28 @@ def build_extension(self, ext: CMakeExtension) -> None: raise cmake_bdist_dir = Path(self.build_temp) / Path(ext.python_package_dir) - copytree( + shutil.copytree( str(cmake_bdist_dir / ext.name), str(Path(output_dir) / ext.name), dirs_exist_ok=True, ) + +class CustomInstall(install): + """Custom handler for the 'install' command.""" + def run(self): + super().run() + bindir = Path(self.build_lib) / "dlite" / "share" / "dlite" / "bin" + # Possible to make a symlink instead of copy to save space + for prog in glob(str(bindir / "*")): + shutil.copy(prog, self.install_scripts) + + version = re.search( r"project\([^)]*VERSION\s+([0-9.]+)", (SOURCE_DIR / "CMakeLists.txt").read_text(), ).groups()[0] +share = Path(".") / "share" / "dlite" setup( name="DLite-Python", @@ -198,20 +213,20 @@ def build_extension(self, ext: CMakeExtension) -> None: install_requires="numpy>=1.14.5,<1.27.0", #install_requires=requirements, #extras_require=extra_requirements, - packages=["dlite"], + packages=["DLite-Python"], scripts=[ str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"), + str(SOURCE_DIR / "cmake" / "patch-activate.sh"), ], package_data={ "dlite": [ - dlite_compiled_ext, - dlite_compiled_dll_suffix, - str(Path(".") / "share" / "dlite" / "storage-plugins" / - dlite_compiled_dll_suffix), - str(Path(".") / "bin" / "dlite-getuuid"), - str(Path(".") / "bin" / "dlite-codegen"), - str(Path(".") / "bin" / "dlite-env"), - str(Path(".") / "bin" / "patch-activate.sh"), + str(share / "README.md"), + str(share / "LICENSE"), + str(share / "storage-plugins" / dlite_compiled_dll_suffix), + str(share / "mapping-plugins" / dlite_compiled_dll_suffix), + str(share / "python-storage-plugins" / "*.py"), + str(share / "python-mapping-plugins" / "*.py"), + str(share / "storages" / "*.json"), ] }, ext_modules=[ @@ -223,6 +238,7 @@ def build_extension(self, ext: CMakeExtension) -> None: ], cmdclass={ "build_ext": CMakeBuildExt, + "install": CustomInstall, }, zip_safe=False, ) From 7524f2f041fa7476e382172b3a874509869f9ebd Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Wed, 4 Sep 2024 21:28:49 +0200 Subject: [PATCH 2/6] Generalised cmake statements by using generator expression --- bindings/python/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index e12a0c60e..b74707010 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -219,9 +219,9 @@ add_custom_command( ${dlite_SOURCE_DIR}/README.md ${dlite_SOURCE_DIR}/LICENSE ${pkgdir}/share/dlite COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_BINARY_DIR}/tools/dlite-codegen${EXEEXT} - ${CMAKE_BINARY_DIR}/tools/dlite-env${EXEEXT} - ${CMAKE_BINARY_DIR}/tools/dlite-getuuid${EXEEXT} + "$" + "$" + "$" ${pkgdir}/share/dlite/bin DEPENDS python_package From 56f2d3aafb940da3a74c658c9cf376bb2ff906d6 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Thu, 5 Sep 2024 23:18:57 +0200 Subject: [PATCH 3/6] Added two more tests --- src/utils/tests/test_uri_encode.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/utils/tests/test_uri_encode.c b/src/utils/tests/test_uri_encode.c index 701332663..3472c0344 100644 --- a/src/utils/tests/test_uri_encode.c +++ b/src/utils/tests/test_uri_encode.c @@ -19,6 +19,16 @@ MU_TEST(test_encode_something) { mu_assert_string_eq("something", buf); mu_assert_int_eq(9, n); } +MU_TEST(test_encode_something_percent) { + int n = uri_encode("something%", 10, buf); + mu_assert_string_eq("something%25", buf); + mu_assert_int_eq(12, n); +} +MU_TEST(test_encode_something_zslash) { + int n = uri_encode("something%z/", 12, buf); + mu_assert_string_eq("something%25z%2F", buf); + mu_assert_int_eq(16, n); +} MU_TEST(test_encode_space) { int n = uri_encode(" ", 1, buf); mu_assert_string_eq("%20", buf); @@ -139,6 +149,8 @@ MU_TEST_SUITE(test_suite) { MU_RUN_TEST(test_encode_empty); MU_RUN_TEST(test_encode_something); + MU_RUN_TEST(test_encode_something_percent); + MU_RUN_TEST(test_encode_something_zslash); MU_RUN_TEST(test_encode_percent); MU_RUN_TEST(test_encode_space); MU_RUN_TEST(test_encode_empty); From ecee45e69c6628c45c048de39ee6602efc1b6631 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Sat, 5 Oct 2024 22:23:58 +0200 Subject: [PATCH 4/6] Added to tips and tricks --- doc/contributors_guide/tips_and_tricks.md | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/doc/contributors_guide/tips_and_tricks.md b/doc/contributors_guide/tips_and_tricks.md index 8f315efcd..39fadcc43 100644 --- a/doc/contributors_guide/tips_and_tricks.md +++ b/doc/contributors_guide/tips_and_tricks.md @@ -6,6 +6,39 @@ Setting up a virtual Python environment for building dlite See [Build against Python environment] in the installation instructions. +Test installation before releasing on PyPI +------------------------------------------ +If you have updated the installation of the [dlite-python] package, or you get failures on GitHub CI/CD that cannot be reproduced locally, you might want to test installing dlite-python before releasing it on PyPI. + +This can be done as follows: + +1. Create a new virtual environment and install requirements and the wheel package + + python -m venv ~/.envs/testenv + source ~/.envs/testenv/bin/activate + pip install -U pip + pip install -r requirements.txt -r requirements_dev.txt wheel + +2. Build the wheel + + cd python + python setup.py bdist_wheel + +3. Install the wheel with pip in a newly created environment (the version numbers may differ for your case) + + pip install dist/DLite_Python-0.5.22-cp311-cp311-linux_x86_64.whl + +4. Finally, test by importing dlite in the standard manner + + cd .. + python + >>> import dlite + + or you can run the python tests + + python bindings/python/tests/test_python_bindings.py + + Debugging Python storage plugins -------------------------------- Exceptions occurring inside Python storage plugins are not propagated to the calling interpreter, and will therefor not be shown. @@ -192,6 +225,7 @@ More useful gdb commands: [virtualenvwrapper]: https://pypi.org/project/virtualenvwrapper/ [Build against Python environment]: https://sintef.github.io/dlite/getting_started/build/build_against_python_env.html#build-against-python-environment +[dlite-python]: https://pypi.org/project/DLite-Python/ [valgrind]: http://valgrind.org/ [gdb]: https://sourceware.org/gdb/ [GDB Tutorial]: https://www.gdbtutorial.com/ From a5960de8acc5ba8d57bd4659903c600a8508723c Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Sun, 6 Oct 2024 09:52:32 +0200 Subject: [PATCH 5/6] Updated plugin paths and and fixed tests accordingly --- bindings/python/dlite-path-python.i | 20 +++++++++++++++++++ .../python/tests/global_dlite_state_mod2.py | 8 ++++---- bindings/python/tests/test_transaction.py | 2 +- doc/contributors_guide/tips_and_tricks.md | 2 +- src/tests/mappings/ent3.json | 7 ++----- src/tests/python/test_python_mapping.c | 4 +++- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/bindings/python/dlite-path-python.i b/bindings/python/dlite-path-python.i index a1ba342e2..e6c6928ac 100644 --- a/bindings/python/dlite-path-python.i +++ b/bindings/python/dlite-path-python.i @@ -43,4 +43,24 @@ mapping_plugin_path = FUPath("mapping-plugins") python_storage_plugin_path = FUPath("python-storage-plugins") python_mapping_plugin_path = FUPath("python-mapping-plugins") +# Update default search paths +from pathlib import Path +pkgdir = Path(__file__).resolve().parent +sharedir = pkgdir / "share" / "dlite" +if (sharedir / "storages").exists(): + storage_path[-1] = sharedir / "storages" + #storage_path.append(sharedir / "storages") +if (sharedir / "storage-plugins").exists(): + storage_plugin_path[-1] = sharedir / "storage-plugins" + #storage_plugin_path.append(sharedir / "storage-plugins") +if (sharedir / "mapping-plugins").exists(): + mapping_plugin_path[-1] = sharedir / "mapping-plugins" + #mapping_plugin_path.append(sharedir / "mapping-plugins") +if (sharedir / "python-storage-plugins").exists(): + python_storage_plugin_path[-1] = sharedir / "python-storage-plugins" + #python_storage_plugin_path.append(sharedir / "python-storage-plugins") +if (sharedir / "python-mapping-plugins").exists(): + python_mapping_plugin_path[-1] = sharedir / "python-mapping-plugins" + #python_mapping_plugin_path.append(sharedir / "python-mapping-plugins") + %} diff --git a/bindings/python/tests/global_dlite_state_mod2.py b/bindings/python/tests/global_dlite_state_mod2.py index de8a8d826..099069461 100644 --- a/bindings/python/tests/global_dlite_state_mod2.py +++ b/bindings/python/tests/global_dlite_state_mod2.py @@ -1,4 +1,7 @@ -#!/usr/bin/env python +# This file is executedevaluated by test_global_dlite_state.py +# +# Note that entitydir is defined in the global scope, so it should +# not be redefined here from pathlib import Path import dlite @@ -7,9 +10,6 @@ assert len(dlite.istore_get_uuids()) == 3 + 3 -thisdir = Path(__file__).absolute().parent -entitydir = thisdir / "entities" - url = f"json://{entitydir}/MyEntity.json" # myentity is already defined via test_global_dlite_state, no new instance is added to istore diff --git a/bindings/python/tests/test_transaction.py b/bindings/python/tests/test_transaction.py index 88c75b824..601267752 100644 --- a/bindings/python/tests/test_transaction.py +++ b/bindings/python/tests/test_transaction.py @@ -7,7 +7,7 @@ # Configure paths thisdir = Path(__file__).parent.absolute() -dlite.storage_path.append(thisdir / "*.json") +dlite.storage_path.append(thisdir / "entities" / "*.json") Person = dlite.get_instance("http://onto-ns.com/meta/0.1/Person") person = Person(dimensions={"N": 4}) diff --git a/doc/contributors_guide/tips_and_tricks.md b/doc/contributors_guide/tips_and_tricks.md index 39fadcc43..7bea6251f 100644 --- a/doc/contributors_guide/tips_and_tricks.md +++ b/doc/contributors_guide/tips_and_tricks.md @@ -17,7 +17,7 @@ This can be done as follows: python -m venv ~/.envs/testenv source ~/.envs/testenv/bin/activate pip install -U pip - pip install -r requirements.txt -r requirements_dev.txt wheel + pip install wheel -r requirements.txt 2. Build the wheel diff --git a/src/tests/mappings/ent3.json b/src/tests/mappings/ent3.json index 48ea31cb8..bb13c28f1 100644 --- a/src/tests/mappings/ent3.json +++ b/src/tests/mappings/ent3.json @@ -1,7 +1,5 @@ { - "name": "ent3", - "version": "0.1", - "namespace": "http://onto-ns.com/meta", + "uri": "http://onto-ns.com/meta/0.1/ent3", "description": "test entity", "dimensions": [], "properties": [ @@ -11,5 +9,4 @@ "unit": "hundreds" } ], - "dataname": "http://onto-ns.com/meta/0.1/ent1" -} \ No newline at end of file +} diff --git a/src/tests/python/test_python_mapping.c b/src/tests/python/test_python_mapping.c index 735800dff..497af3476 100644 --- a/src/tests/python/test_python_mapping.c +++ b/src/tests/python/test_python_mapping.c @@ -42,12 +42,14 @@ MU_TEST(test_initialize) MU_TEST(test_map) { - DLiteInstance *insts[1], *inst3; + DLiteInstance *insts[1], *inst3, *ent3; const DLiteInstance **instances = (const DLiteInstance **)insts; void *p; instances[0] = dlite_instance_get("2daa6967-8ecd-4248-97b2-9ad6fefeac14"); mu_check(instances[0]); + ent3 = dlite_instance_get("http://onto-ns.com/meta/0.1/ent3"); + mu_check(ent3); inst3 = dlite_mapping("http://onto-ns.com/meta/0.1/ent3", instances, 1); mu_check(inst3); mu_check((p = dlite_instance_get_property(inst3, "c"))); From f15d503d28898e653fec3409a23a81ddb8659041 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Mon, 7 Oct 2024 12:36:55 +0200 Subject: [PATCH 6/6] Update bindings/python/tests/global_dlite_state_mod2.py Co-authored-by: Francesca L. Bleken <48128015+francescalb@users.noreply.github.com> --- bindings/python/tests/global_dlite_state_mod2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/tests/global_dlite_state_mod2.py b/bindings/python/tests/global_dlite_state_mod2.py index 099069461..567aafc75 100644 --- a/bindings/python/tests/global_dlite_state_mod2.py +++ b/bindings/python/tests/global_dlite_state_mod2.py @@ -1,4 +1,4 @@ -# This file is executedevaluated by test_global_dlite_state.py +# This file is executed by test_global_dlite_state.py # # Note that entitydir is defined in the global scope, so it should # not be redefined here