LLVM: Initial CPython Interop example

.github/workflows/CI.yml

@@ -396,8 +396,8 @@ jobs:
         shell: bash -e -l {0}
         run: |
             cd integration_tests
-            ./run_tests.py -b cpython c_py
-            ./run_tests.py -b cpython c_py -f
+            ./run_tests.py -b cpython c_py llvm_py
+            ./run_tests.py -b cpython c_py llvm_py -f
 
   sympy:
     name: Run SymPy tests

integration_tests/CMakeLists.txt

@@ -76,13 +76,14 @@ message("LPYTHON_RTLIB_DIR: ${LPYTHON_RTLIB_DIR}")
 message("LPYTHON_RTLIB_LIBRARY: ${LPYTHON_RTLIB_LIBRARY}")
 
 
-macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS)
+macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS RUN_COPY_TO_BIN)
     set(fail ${${RUN_FAIL}})
     set(name ${${RUN_NAME}})
     set(file_name ${${RUN_FILE_NAME}})
     set(labels ${${RUN_LABELS}})
     set(extra_files ${${RUN_EXTRAFILES}})
     set(extra_args ${${RUN_EXTRA_ARGS}})
+    set(copy_to_bin ${${RUN_COPY_TO_BIN}})
 
     if (NOT name)
         message(FATAL_ERROR "Must specify the NAME argument")
@@ -105,6 +106,23 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXT
             if (${fail})
                 set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE)
             endif()
+        elseif (KIND STREQUAL "llvm_py")
+            add_custom_command(
+                OUTPUT ${name}.o
+                COMMAND ${LPYTHON} -c ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py -o ${name}.o
+                DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}.py
+                VERBATIM)
+            add_executable(${name} ${name}.o ${extra_files})
+            target_include_directories(${name} PRIVATE ${CMAKE_SOURCE_DIR} ${NUMPY_INCLUDE_DIR})
+            set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C)
+            target_link_libraries(${name} lpython_rtlib Python::Python)
+            add_test(${name} ${CMAKE_CURRENT_BINARY_DIR}/${name})
+            if (labels)
+                set_tests_properties(${name} PROPERTIES LABELS "${labels}")
+            endif()
+            if (${fail})
+                set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE)
+            endif()
         elseif(KIND STREQUAL "llvm_sym")
             add_custom_command(
                 OUTPUT ${name}.o
@@ -153,9 +171,6 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXT
             target_include_directories(${name} PRIVATE ${CMAKE_SOURCE_DIR} ${NUMPY_INCLUDE_DIR})
             set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C)
             target_link_libraries(${name} lpython_rtlib Python::Python)
-            if (extra_files)
-                file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${extra_files} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-            endif()
             add_test(${name} ${CMAKE_CURRENT_BINARY_DIR}/${name})
             if (labels)
                 set_tests_properties(${name} PROPERTIES LABELS "${labels}")
@@ -281,12 +296,17 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXT
                 set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE)
             endif()
         endif()
+
+        if (copy_to_bin)
+            file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${copy_to_bin} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+        endif()
+
     endif()
 endmacro(RUN_UTIL)
 
 macro(RUN)
     set(options FAIL NOFAST ENABLE_CPYTHON LINK_NUMPY)
-    set(oneValueArgs NAME IMPORT_PATH)
+    set(oneValueArgs NAME IMPORT_PATH COPY_TO_BIN)
     set(multiValueArgs LABELS EXTRAFILES)
     cmake_parse_arguments(RUN "${options}" "${oneValueArgs}"
                         "${multiValueArgs}" ${ARGN} )
@@ -309,14 +329,14 @@ macro(RUN)
     endif()
 
     if (NOT FAST)
-        RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS)
+        RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS RUN_COPY_TO_BIN)
     endif()
 
     if ((FAST) AND (NOT RUN_NOFAST))
         set(RUN_EXTRA_ARGS ${RUN_EXTRA_ARGS} --fast)
         set(RUN_NAME "${RUN_NAME}_FAST")
         list(REMOVE_ITEM RUN_LABELS cpython cpython_sym) # remove cpython, cpython_sym, from --fast test
-        RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS)
+        RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS RUN_COPY_TO_BIN)
     endif()
 endmacro(RUN)
 
@@ -575,10 +595,11 @@ RUN(NAME bindc_05   LABELS llvm c
         EXTRAFILES bindc_05b.c)
 RUN(NAME bindc_06   LABELS llvm c
         EXTRAFILES bindc_06b.c)
-RUN(NAME bindpy_01           LABELS cpython c_py ENABLE_CPYTHON NOFAST EXTRAFILES bindpy_01_module.py)
-RUN(NAME bindpy_02           LABELS cpython c_py LINK_NUMPY EXTRAFILES bindpy_02_module.py)
-RUN(NAME bindpy_03           LABELS cpython c_py LINK_NUMPY NOFAST EXTRAFILES bindpy_03_module.py)
-RUN(NAME bindpy_04           LABELS cpython c_py LINK_NUMPY NOFAST EXTRAFILES bindpy_04_module.py)
+RUN(NAME bindpy_01           LABELS cpython c_py ENABLE_CPYTHON NOFAST COPY_TO_BIN bindpy_01_module.py)
+RUN(NAME bindpy_02           LABELS cpython c_py LINK_NUMPY COPY_TO_BIN bindpy_02_module.py)
+RUN(NAME bindpy_03           LABELS cpython c_py LINK_NUMPY NOFAST COPY_TO_BIN bindpy_03_module.py)
+RUN(NAME bindpy_04           LABELS cpython c_py LINK_NUMPY NOFAST COPY_TO_BIN bindpy_04_module.py)
+RUN(NAME bindpy_05           LABELS llvm_py c_py ENABLE_CPYTHON COPY_TO_BIN bindpy_05_module.py)
 RUN(NAME test_generics_01    LABELS cpython llvm c NOFAST)
 RUN(NAME test_cmath          LABELS cpython llvm c NOFAST)
 RUN(NAME test_complex_01        LABELS cpython llvm c wasm wasm_x64)

integration_tests/bindpy_05.py

@@ -0,0 +1,79 @@
+from lpython import ccall, Pointer, i32, i64, empty_c_void_p, CPtr, pointer
+
+@ccall(header="Python.h")
+def Py_Initialize():
+    pass
+
+@ccall(header="Python.h")
+def Py_DecodeLocale(s: str, p: CPtr) -> CPtr:
+    pass
+
+@ccall(header="Python.h")
+def PySys_SetArgv(n: i32, args: Pointer[CPtr]):
+    pass
+
+@ccall(header="Python.h")
+def Py_FinalizeEx() -> i32:
+    pass
+
+@ccall(header="Python.h")
+def PyUnicode_FromString(s: str) -> CPtr:
+    pass
+
+@ccall(header="Python.h")
+def PyImport_Import(name: CPtr) -> CPtr:
+    pass
+
+@ccall(header="Python.h")
+def _Py_DecRef(name: CPtr):
+    pass
+
+@ccall(header="Python.h")
+def PyObject_GetAttrString(m: CPtr, s: str) -> CPtr:
+    pass
+
+@ccall(header="Python.h")
+def PyTuple_New(n: i32) -> CPtr:
+    pass
+
+@ccall(header="Python.h")
+def PyObject_CallObject(a: CPtr, b: CPtr) -> CPtr:
+    pass
+
+@ccall(header="Python.h")
+def PyLong_AsLongLong(a: CPtr) -> i32:
+    pass
+
+def my_f():
+    pName: CPtr; pModule: CPtr; pFunc: CPtr; pArgs: CPtr; pValue: CPtr
+
+    pName = PyUnicode_FromString("bindpy_05_module")
+    assert bool(pName), "Failed to convert to unicode string bindpy_05_module\n"
+
+    pModule = PyImport_Import(pName)
+    _Py_DecRef(pName)
+    assert bool(pModule), "Failed to load python module bindpy_05_module\n"
+
+    pFunc = PyObject_GetAttrString(pModule, "my_f")
+    assert bool(pFunc), "Cannot find function my_f\n"
+
+    pArgs = PyTuple_New(0)
+    pValue = PyObject_CallObject(pFunc, pArgs)
+    _Py_DecRef(pArgs)
+    assert bool(pValue), "Call to my_f failed\n"
+
+    ans: i32 = PyLong_AsLongLong(pValue)
+    print("Ans is", ans)
+    assert ans == 5
+
+
+def main0():
+    Py_Initialize()
+    argv1: CPtr = Py_DecodeLocale("", empty_c_void_p())
+    PySys_SetArgv(1, pointer(argv1, i64))
+
+    my_f()
+
+    assert(Py_FinalizeEx() >= 0), "BindPython: Unknown Error in FinalizeEx()\n"
+
+main0()

integration_tests/bindpy_05_module.py

@@ -0,0 +1,3 @@
+def my_f():
+    print("hello from python")
+    return 5

integration_tests/run_tests.py

@@ -6,7 +6,7 @@
 
 # Initialization
 DEFAULT_THREADS_TO_USE = 8 # default no of threads is 8
-SUPPORTED_BACKENDS = ['llvm', 'c', 'wasm', 'cpython', 'x86', 'wasm_x86', 'wasm_x64', 'c_py', 'c_sym', 'cpython_sym', 'llvm_sym']
+SUPPORTED_BACKENDS = ['llvm', 'c', 'wasm', 'cpython', 'x86', 'wasm_x86', 'wasm_x64', 'c_py', 'c_sym', 'cpython_sym', 'llvm_sym', 'llvm_py']
 BASE_DIR = os.path.dirname(os.path.realpath(__file__))
 LPYTHON_PATH = f"{BASE_DIR}/../src/bin"
 
@@ -62,7 +62,7 @@ def main():
     DEFAULT_THREADS_TO_USE = args.no_of_threads or DEFAULT_THREADS_TO_USE
     fast_tests = "yes" if args.fast else "no"
     for backend in args.backends:
-        python_libs_req = "yes" if backend in ["c_py", "c_sym", "llvm_sym"] else "no"
+        python_libs_req = "yes" if backend in ["c_py", "c_sym", "llvm_sym", 'llvm_py'] else "no"
         test_backend(backend)
 
 

src/bin/lpython.cpp

@@ -1226,6 +1226,16 @@ int link_executable(const std::vector<std::string> &infiles,
             if (compiler_options.enable_symengine) {
                 cmd += " -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lsymengine";
             }
+
+            if (compiler_options.enable_cpython) {
+                std::string py_version = "3.10";
+                std::string py_flags = R"(-I $CONDA_PREFIX/include/python)" + py_version + R"( -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lpython)" + py_version + R"()";
+                if (compiler_options.link_numpy) {
+                    py_flags += R"( -I$CONDA_PREFIX/lib/python)" + py_version + R"(/site-packages/numpy/core/include)";
+                }
+                cmd += " " + py_flags;
+            }
+
             int err = system(cmd.c_str());
             if (err) {
                 std::cout << "The command '" + cmd + "' failed." << std::endl;

src/lpython/semantics/python_intrinsic_eval.h

@@ -244,6 +244,10 @@ struct IntrinsicNodeHandler {
 
         } else if (ASRUtils::is_logical(*type)) {
             return (ASR::asr_t *)arg;
+        } else if (ASR::is_a<ASR::CPtr_t>(*type)) {
+            ASR::expr_t* c_null_ptr = ASRUtils::EXPR(ASR::make_PointerNullConstant_t(
+                    al, loc, ASRUtils::TYPE(ASR::make_CPtr_t(al, loc))));
+            return ASR::make_CPtrCompare_t(al, loc, arg, ASR::cmpopType::NotEq, c_null_ptr, to_type, nullptr);
         } else {
             std::string stype = ASRUtils::type_to_str_python(type);
             throw SemanticError(