conan.cmake fails when using pyenv

[BlueSolei]

Issue

Hi,
The conan.cmake script fails due to the conan command is not an executable.
I use PyEnv on windows, so my conan command is a .bat file, or if running in Git-Bash, its a sh script.

The error I get is Conan install failed='%1 is not a valid Win32 application
Here is the cmake run:

√ /c/dev/expr/conan-training/requires_conditional (shaul) $ cmake -B build -S src/
-- Building for: Visual Studio 16 2019
-- The CXX compiler identification is MSVC 19.27.29110.0
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: C:/Program Files (x86)/Microsoft Visual Studio/2019/Professional/VC/Tools/MSVC/14.27.29110/bin/Hostx64/x64/cl.exe - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Conan: Using cmake-multi generator
-- Conan: Automatic detection of conan settings from cmake
-- Conan: Detected VS runtime: MD
-- Conan: Settings= -s;arch=x86_64;-s;build_type=Release;-s;compiler=Visual Studio;-s;compiler.version=16;-s;compiler.runtime=MD
-- Conan: checking conan executable
-- Conan: Found program C:/Users/sfridman/.pyenv/pyenv-win/shims/conan
-- Conan: Version found
-- Conan executing: C:/Users/sfridman/.pyenv/pyenv-win/shims/conan install . -s arch=x86_64 -s build_type=Release -s compiler=Visual Studio -s compiler.version=16 -s compiler.runtime=MD -g=cmake_multi
CMake Error at C:/dev/expr/cmake-conan/conan.cmake:455 (message):
  Conan install failed='%1 is not a valid Win32 application'
Call Stack (most recent call first):
  C:/dev/expr/cmake-conan/conan.cmake:540 (conan_cmake_install)
  CMakeLists.txt:5 (conan_cmake_run)


-- Configuring incomplete, errors occurred!
See also "C:/dev/expr/conan-training/requires_conditional/build/CMakeFiles/CMakeOutput.log".

A fix

I manage to fix the issue by running the calls to conan in a shell, like this:

set(SHELL_CONAN_CMD "cmd.exe")
set(SHELL_CONAN_ARGS "/C;${CONAN_CMD};${conan_args}")    

string (REPLACE ";" " " _conan_args "${SHELL_CONAN_ARGS}")
message(STATUS "Conan executing: ${SHELL_CONAN_CMD} ${_conan_args}")

if(ARGUMENTS_OUTPUT_QUIET)
  execute_process(COMMAND ${SHELL_CONAN_CMD} ${SHELL_CONAN_ARGS}
                   RESULT_VARIABLE return_code
                   OUTPUT_VARIABLE conan_output
                   ERROR_VARIABLE conan_output
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
else()
  execute_process(COMMAND ${SHELL_CONAN_CMD} ${SHELL_CONAN_ARGS}
                   RESULT_VARIABLE return_code
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
endif()

I am quite a novice in CMake, so I don't know what will be the right way to fix this.

[czoido]

Hi @BlueSolei,
Thanks a lot for reporting and proposing the fix, we'll try to reproduce and fix the issue.

[valerijbielskij]

Has this issue been fixed? I still face it with v0.16.1.

[BlueSolei]

I opened this issue, but after using Conan for a while, I understood that the right way to go about here is to use python's virtual environment.
This way you don't care about pyenv, as inside the virtual environment you have python.exe which works great :-)

[valerijbielskij]

Thanks for the reply, I'm trying to run it over msys2, starting python's virtual env didn't help.

[BlueSolei]

Do you mean git bash?
I had issues with activating virtualenv on git-bash.
Don't remember how I fixed it though :-|
Anyway, with this solution attempt the issue is no more conan+pyenv, but virtualenv on windows msys.
You need to make sure the virtualenv is activated, and when you do where conan you get the one in the venv (install conan in the venv with pip).

[valerijbielskij]

that worked, thanks a lot!

[maldag]

I opened this issue, but after using Conan for a while, I understood that the right way to go about here is to use python's virtual environment. This way you don't care about pyenv, as inside the virtual environment you have python.exe which works great :-)

This is not always correct since you can create a virtualenv with activated pyenv-shell. That way you can also guarantee which python version will be used for virtualenv. Virtualenv just isolates python packages, not python versions.

So using this snippet will also cause problems:

pyenv local 3.x.y
pyenv shell python -m venv. env # force creation of virtualenv with given pyenv version 3.x.y
.env\scripts\activate
# now in virtualenv
pip install -U conan
conan install ... # whatever

[BlueSolei]

I opened this issue, but after using Conan for a while, I understood that the right way to go about here is to use python's virtual environment. This way you don't care about pyenv, as inside the virtual environment you have python.exe which works great :-)

This is not always correct since you can create a virtualenv with activated pyenv-shell. That way you can also guarantee which python version will be used for virtualenv. Virtualenv just isolates python packages, not python versions.

So using this snippet will also cause problems:

pyenv local 3.x.y
pyenv shell python -m venv. env # force creation of virtualenv with given pyenv version 3.x.y
.env\scripts\activate
# now in virtualenv
pip install -U conan
conan install ... # whatever

venv does have its own python.exe
so when you are in the venv you have python.exe so the cmake-conan will run successfully.
to check this, run where python and make sure it is the one inside the venv.

[maldag]

I opened this issue, but after using Conan for a while, I understood that the right way to go about here is to use python's virtual environment. This way you don't care about pyenv, as inside the virtual environment you have python.exe which works great :-)

This is not always correct since you can create a virtualenv with activated pyenv-shell. That way you can also guarantee which python version will be used for virtualenv. Virtualenv just isolates python packages, not python versions.
So using this snippet will also cause problems:

pyenv local 3.x.y
pyenv shell python -m venv. env # force creation of virtualenv with given pyenv version 3.x.y
.env\scripts\activate
# now in virtualenv
pip install -U conan
conan install ... # whatever

venv does have its own python.exe so when you are in the venv you have python.exe so the cmake-conan will run successfully. to check this, run where python and make sure it is the one inside the venv.

Ok, sure does it have it's own python.exe, but either I messed up with PATH variable on windows, which in fact is needed to be prepend pyenv-win/shims path to get pyenv working in the first place, or I'm missing something else.

# Output of conan install ..
[...]
Program python3 found: YES (C:\Users\<USERNAME>\.pyenv\pyenv-win\shims\python3.BAT)

source_subfolder\meson.build:2188:0: ERROR: <ExternalProgram 'python3' -> ['C:\\Users\\<USERNAME>\\.pyenv\\pyenv-win\\shims\\python3.BAT']> is not a valid python or it is missing setuptools

A full log can be found at C:\.conan\069d7a\1\build_subfolder\meson-logs\meson-log.txt

So it does not compile in this case for me since meson build is not looking up python.exe in virtualenv apparently.

[BlueSolei]

I don't know why Conan is looking for python3 :-|

After activating the venv (.env\scripts\activate)
Can you please put here the output of where python?
this line

Program python3 found: YES (C:\Users<USERNAME>.pyenv\pyenv-win\shims\python3.BAT)

Shows that it looking for python3 which is wrong in the first place.

[maldag]

# Output of `where python`
(.env) D:\xyz\conan-qt-test>where python
D:\xyz\conan-qt-test\.env\Scripts\python.exe
C:\Program Files\Python39\python.exe
C:\Users\<USERNAME>\.pyenv\pyenv-win\shims\python
C:\Users\<USERNAME>\.pyenv\pyenv-win\shims\python.bat
C:\Users\<USERNAME>\AppData\Local\Microsoft\WindowsApps\python.exe

I'm using either cmd.exe on windows 10 or powershell version 7.2

[earonesty]

isolating in a venv worked for me, because when you install conan, it goes into scripts/conan.exe, and that shows up first in the path

#!/bin/sh
python -mvirtualenv venv
. venv/Scripts/activate 2>/dev/null || . venv/bin/activate
python -mpip install --upgrade conan

# cmake stuff goes here

shims show up after venv:

$ where conan
C:\Users\XXX\git\curvekata\venv\Scripts\conan.exe
C:\Users\XXX\.pyenv\pyenv-win\shims\conan
C:\Users\XXX\.pyenv\pyenv-win\shims\conan.bat

[maldag]

Just bumping on this issue again as it doesn't seemed to be fixed for me yet. We're using Windows 10, and it's still looking for python3 in the build process even though it was started from the command line with activated venv

The log file from the failing build job looks like this:

Checking if "/proc/self/cmdline" runs: DID NOT COMPILE
Program python3 found: YES (C:\Users\<Username>\.pyenv\pyenv-win\shims\python3.BAT)
Could not introspect Python (['C:\\Users\\<Username>\\.pyenv\\pyenv-win\\shims\\python3.BAT', '-c', "import sysconfig\nimport json\nimport sys\n\ninstall_paths = sysconfig.get_paths(scheme='posix_prefix', vars={'base': '', 'platbase': '', 'installed_base': ''})\n\ndef links_against_libpython():\n    from distutils.core import Distribution, Extension\n    cmd = Distribution().get_command_obj('build_ext')\n    cmd.ensure_finalized()\n    return bool(cmd.get_libraries(Extension('dummy', [])))\n\nprint (json.dumps ({\n  'variables': sysconfig.get_config_vars(),\n  'paths': sysconfig.get_paths(),\n  'install_paths': install_paths,\n  'version': sysconfig.get_python_version(),\n  'platform': sysconfig.get_platform(),\n  'is_pypy': '__pypy__' in sys.builtin_module_names,\n  'link_libpython': links_against_libpython(),\n}))\n"]): exit code 0
Program stdout:

No global python version has been set yet. Please set the global version by typing:
pyenv global 3.7.2

Program stderr:

Running it again yields to:

Could not compile test file C:\.conan\069d7a\1\build_subfolder\meson-private\tmpl9jw1l8i\testfile.c: 2

Checking if "/proc/self/cmdline" runs: DID NOT COMPILE
Program python3 found: YES (C:\Users\<Username>\.pyenv\pyenv-win\shims\python3.BAT)
Could not introspect Python (['C:\\Users\\<Username>\\.pyenv\\pyenv-win\\shims\\python3.BAT', '-c', "import sysconfig\nimport json\nimport sys\n\ninstall_paths = sysconfig.get_paths(scheme='posix_prefix', vars={'base': '', 'platbase': '', 'installed_base': ''})\n\ndef links_against_libpython():\n    from distutils.core import Distribution, Extension\n    cmd = Distribution().get_command_obj('build_ext')\n    cmd.ensure_finalized()\n    return bool(cmd.get_libraries(Extension('dummy', [])))\n\nprint (json.dumps ({\n  'variables': sysconfig.get_config_vars(),\n  'paths': sysconfig.get_paths(),\n  'install_paths': install_paths,\n  'version': sysconfig.get_python_version(),\n  'platform': sysconfig.get_platform(),\n  'is_pypy': '__pypy__' in sys.builtin_module_names,\n  'link_libpython': links_against_libpython(),\n}))\n"]): exit code 0
Program stdout:


Program stderr:



source_subfolder\meson.build:2188:0: ERROR: <ExternalProgram 'python3' -> ['C:\\Users\\<Username>\\.pyenv\\pyenv-win\\shims\\python3.BAT']> is not a valid python or it is missing setuptools

[garyo]

I use this to find the "real" conan when it's actually a shell script with pyenv, in case this is useful to anyone:

function(set_conan_cmd)
    if (DEFINED CONAN_CMD)
       return()
    endif()
    set(in_conan_cmd ${ARGN})

    # If in_conan_cmd is empty, use "conan"
    if ((NOT DEFINED in_conan_cmd) OR (${in_conan_cmd} EQUAL ""))
        set(in_conan_cmd conan)
    endif()

    # If in_conan_cmd is not an absolute path, find it
    if (NOT (${in_conan_cmd} MATCHES "^/" OR ${in_conan_cmd} MATCHES "^[a-zA-Z]:/"))
        find_program(in_conan_cmd ${in_conan_cmd})
    endif()

    # On Windows, check for pyenv script and find the real binary
    if (WIN32 AND ${in_conan_cmd} MATCHES ".*pyenv")
        find_program(pyenv_cmd pyenv)
        execute_process(COMMAND "c:/bin/bash" ${pyenv_cmd} which conan
                        OUTPUT_VARIABLE conan_path
                        OUTPUT_STRIP_TRAILING_WHITESPACE)
        file(TO_CMAKE_PATH ${conan_path} conan_path) # normalize backslashes to forward
        set(CONAN_CMD ${conan_path} PARENT_SCOPE)
        message(STATUS "Found pyenv conan (Win32) at ${conan_path}")
    else()
        file(TO_CMAKE_PATH ${in_conan_cmd} in_conan_cmd)
        set(CONAN_CMD ${in_conan_cmd} PARENT_SCOPE)
    endif()
endfunction()