From 604ae6fdfd91d62e44caeb6f6ef32209bf6ec92e Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Date: Thu, 26 Oct 2017 17:17:16 +0200 Subject: [PATCH 01/38] Merged 0.28.0 --- conans/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index 7963bf2c49d..af300fc7594 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -17,4 +17,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.28.0' +__version__ = '0.29.0-dev' From 3ad621836fdaa9456c00ba3feee529e78a6e97f8 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Fri, 27 Oct 2017 11:45:29 +0200 Subject: [PATCH 02/38] Added tests (#1948) --- conans/client/profile_loader.py | 2 +- conans/test/integration/profile_test.py | 2 +- conans/test/model/profile_test.py | 9 +++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index 539f65ad105..fbbd9fa924f 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -22,7 +22,7 @@ def __init__(self, text): self.profile_text = "" for counter, line in enumerate(text.splitlines()): - if not line or line.strip().startswith("#"): + if not line.strip() or line.strip().startswith("#"): continue elif line.strip().startswith("["): self.profile_text = "\n".join(text.splitlines()[counter:]) diff --git a/conans/test/integration/profile_test.py b/conans/test/integration/profile_test.py index 33272bf4f61..ef977021b80 100644 --- a/conans/test/integration/profile_test.py +++ b/conans/test/integration/profile_test.py @@ -38,7 +38,7 @@ def create_profile(folder, name, settings=None, scopes=None, package_settings=No package_env=None, options=None): _create_profile(folder, name, settings, scopes, package_settings, env, package_env, options) content = load(os.path.join(folder, name)) - content = "include(default)\n" + content + content = "include(default)\n \n" + content save(os.path.join(folder, name), content) diff --git a/conans/test/model/profile_test.py b/conans/test/model/profile_test.py index 8e85599ba32..9aa3a08a11f 100644 --- a/conans/test/model/profile_test.py +++ b/conans/test/model/profile_test.py @@ -1,14 +1,9 @@ -import os import unittest -from conans.client.profile_loader import _load_profile, read_profile +from conans.client.profile_loader import _load_profile from conans.model.profile import Profile from collections import OrderedDict -from conans.model.ref import ConanFileReference -from conans.test.utils.test_files import temp_folder -from conans.util.files import save - class ProfileTest(unittest.TestCase): @@ -50,6 +45,8 @@ def profile_subsettings_update_test(self): def package_settings_update_test(self): prof = '''[settings] MyPackage:os=Windows + + # In the previous line there are some spaces ''' np, _ = _load_profile(prof, None, None) From 7a9a0dc6f9733067c38afc86a83ce4de2c8e01a8 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 27 Oct 2017 14:25:35 +0200 Subject: [PATCH 03/38] fixed help message --- conans/client/command.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conans/client/command.py b/conans/client/command.py index bb5c2fe3a34..b4a2bda7f49 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -1191,8 +1191,7 @@ def _add_common_install_arguments(parser, build_help): help='Apply the specified profile to the install command') parser.add_argument("-r", "--remote", help='look in the specified remote server') parser.add_argument("--options", "-o", - help='Options to build the package, overwriting the defaults. e.g., ' - '-o with_qt=true', + help='Define options values, e.g., -o Pkg:with_qt=true', nargs=1, action=Extender) parser.add_argument("--settings", "-s", help='Settings to build the package, overwriting the defaults. e.g., ' From 38e0c3fcb8a9f19d0e2d8da15941f56e0baba33d Mon Sep 17 00:00:00 2001 From: James Date: Fri, 27 Oct 2017 14:47:09 +0200 Subject: [PATCH 04/38] refactor client folder layout (#1949) * refactor client folder layout * split build envs * fixed import * excluded folder --- .gitignore | 2 + conans/__init__.py | 12 ++--- conans/client/build/__init__.py | 0 .../autotools_environment.py} | 52 +++---------------- conans/client/{ => build}/cmake.py | 0 .../{ => build}/configure_environment.py | 0 conans/client/{ => build}/gcc.py | 0 conans/client/{ => build}/meson.py | 0 conans/client/build/visual_environment.py | 48 +++++++++++++++++ conans/client/client_cache.py | 2 +- conans/client/conan_api.py | 2 +- conans/client/{ => conf}/detect.py | 0 conans/client/generators/gcc.py | 2 +- conans/client/generators/virtualbuildenv.py | 3 +- conans/client/manager.py | 4 +- conans/test/command/install_test.py | 2 +- .../functional/autotools_configure_test.py | 2 +- conans/test/functional/cmake_test.py | 2 +- .../test/functional/compile_helpers_test.py | 4 +- conans/test/integration/cmake_multi_test.py | 2 +- .../integration/conf_default_settings_test.py | 2 +- conans/test/util/detect_test.py | 2 +- 22 files changed, 76 insertions(+), 67 deletions(-) create mode 100644 conans/client/build/__init__.py rename conans/client/{configure_build_environment.py => build/autotools_environment.py} (85%) rename conans/client/{ => build}/cmake.py (100%) rename conans/client/{ => build}/configure_environment.py (100%) rename conans/client/{ => build}/gcc.py (100%) rename conans/client/{ => build}/meson.py (100%) create mode 100644 conans/client/build/visual_environment.py rename conans/client/{ => conf}/detect.py (100%) diff --git a/.gitignore b/.gitignore index ae103f0694d..56c4835e83a 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,5 @@ cacert.pem .conan_server/ .sudo_as_admin_successful +# add excluded +!conans/client/build \ No newline at end of file diff --git a/conans/__init__.py b/conans/__init__.py index af300fc7594..965d43b3a33 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -3,14 +3,14 @@ from conans.model.conan_file import ConanFile from conans.model.options import Options from conans.model.settings import Settings -from conans.client.cmake import CMake -from conans.client.meson import Meson -from conans.client.gcc import GCC -from conans.client.configure_environment import ConfigureEnvironment -from conans.client.configure_build_environment import (AutoToolsBuildEnvironment, VisualStudioBuildEnvironment) +from conans.client.build.cmake import CMake +from conans.client.build.meson import Meson +from conans.client.build.gcc import GCC +from conans.client.build.configure_environment import ConfigureEnvironment +from conans.client.build.autotools_environment import AutoToolsBuildEnvironment +from conans.client.build.visual_environment import VisualStudioBuildEnvironment from conans.client.run_environment import RunEnvironment from conans.util.files import load -import os # complex_search: With ORs and not filtering by not restricted settings COMPLEX_SEARCH_CAPABILITY = "complex_search" diff --git a/conans/client/build/__init__.py b/conans/client/build/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/client/configure_build_environment.py b/conans/client/build/autotools_environment.py similarity index 85% rename from conans/client/configure_build_environment.py rename to conans/client/build/autotools_environment.py index 9dd3eff7417..cf877891546 100644 --- a/conans/client/configure_build_environment.py +++ b/conans/client/build/autotools_environment.py @@ -37,46 +37,6 @@ def stdlib_defines(compiler, libcxx): return ret -class VisualStudioBuildEnvironment(object): - """ - - LIB: library paths with semicolon separator - - CL: /I (include paths) - """ - def __init__(self, conanfile): - """ - :param conanfile: ConanFile instance - :param quote_paths: The path directories will be quoted. If you are using the vars together with - environment_append keep it to True, for virtualbuildenv quote_paths=False is required. - """ - self.include_paths = conanfile.deps_cpp_info.include_paths - self.lib_paths = conanfile.deps_cpp_info.lib_paths - - @property - def vars(self): - """Used in conanfile with environment_append""" - cl_args = " ".join(['/I"%s"' % lib for lib in self.include_paths]) + environ_value_prefix("CL") - lib_paths = ";".join(['%s' % lib for lib in self.lib_paths]) + environ_value_prefix("LIB", ";") - return {"CL": cl_args, - "LIB": lib_paths} - - @property - def vars_dict(self): - """Used in virtualbuildenvironment""" - # Here we do not quote the include paths, it's going to be used by virtual environment - cl = ['/I%s' % lib for lib in self.include_paths] - lib = [lib for lib in self.lib_paths] # copy - - if os.environ.get("CL", None): - cl.append(os.environ.get("CL")) - - if os.environ.get("LIB", None): - lib.append(os.environ.get("LIB")) - - ret = {"CL": cl, - "LIB": lib} - return ret - - class AutoToolsBuildEnvironment(object): """ - CPPFLAGS (C-PreProcesor-Flags NOT related with c++) (-I -D) @@ -299,11 +259,11 @@ def vars(self): ld_flags, cpp_flags, libs, cxx_flags, c_flags = self._get_vars() - cpp_flags = " ".join(cpp_flags) + environ_value_prefix("CPPFLAGS") - cxx_flags = " ".join(cxx_flags) + environ_value_prefix("CXXFLAGS") - cflags = " ".join(c_flags) + environ_value_prefix("CFLAGS") - ldflags = " ".join(ld_flags) + environ_value_prefix("LDFLAGS") - libs = " ".join(libs) + environ_value_prefix("LIBS") + cpp_flags = " ".join(cpp_flags) + _environ_value_prefix("CPPFLAGS") + cxx_flags = " ".join(cxx_flags) + _environ_value_prefix("CXXFLAGS") + cflags = " ".join(c_flags) + _environ_value_prefix("CFLAGS") + ldflags = " ".join(ld_flags) + _environ_value_prefix("LDFLAGS") + libs = " ".join(libs) + _environ_value_prefix("LIBS") ret = {"CPPFLAGS": cpp_flags.strip(), "CXXFLAGS": cxx_flags.strip(), @@ -315,7 +275,7 @@ def vars(self): return ret -def environ_value_prefix(var_name, prefix=" "): +def _environ_value_prefix(var_name, prefix=" "): if os.environ.get(var_name, ""): return "%s%s" % (prefix, os.environ.get(var_name, "")) else: diff --git a/conans/client/cmake.py b/conans/client/build/cmake.py similarity index 100% rename from conans/client/cmake.py rename to conans/client/build/cmake.py diff --git a/conans/client/configure_environment.py b/conans/client/build/configure_environment.py similarity index 100% rename from conans/client/configure_environment.py rename to conans/client/build/configure_environment.py diff --git a/conans/client/gcc.py b/conans/client/build/gcc.py similarity index 100% rename from conans/client/gcc.py rename to conans/client/build/gcc.py diff --git a/conans/client/meson.py b/conans/client/build/meson.py similarity index 100% rename from conans/client/meson.py rename to conans/client/build/meson.py diff --git a/conans/client/build/visual_environment.py b/conans/client/build/visual_environment.py new file mode 100644 index 00000000000..e5a62965f55 --- /dev/null +++ b/conans/client/build/visual_environment.py @@ -0,0 +1,48 @@ +import os + + +class VisualStudioBuildEnvironment(object): + """ + - LIB: library paths with semicolon separator + - CL: /I (include paths) + """ + def __init__(self, conanfile): + """ + :param conanfile: ConanFile instance + :param quote_paths: The path directories will be quoted. If you are using the vars together with + environment_append keep it to True, for virtualbuildenv quote_paths=False is required. + """ + self.include_paths = conanfile.deps_cpp_info.include_paths + self.lib_paths = conanfile.deps_cpp_info.lib_paths + + @property + def vars(self): + """Used in conanfile with environment_append""" + cl_args = " ".join(['/I"%s"' % lib for lib in self.include_paths]) + _environ_value_prefix("CL") + lib_paths = ";".join(['%s' % lib for lib in self.lib_paths]) + _environ_value_prefix("LIB", ";") + return {"CL": cl_args, + "LIB": lib_paths} + + @property + def vars_dict(self): + """Used in virtualbuildenvironment""" + # Here we do not quote the include paths, it's going to be used by virtual environment + cl = ['/I%s' % lib for lib in self.include_paths] + lib = [lib for lib in self.lib_paths] # copy + + if os.environ.get("CL", None): + cl.append(os.environ.get("CL")) + + if os.environ.get("LIB", None): + lib.append(os.environ.get("LIB")) + + ret = {"CL": cl, + "LIB": lib} + return ret + + +def _environ_value_prefix(var_name, prefix=" "): + if os.environ.get(var_name, ""): + return "%s%s" % (prefix, os.environ.get(var_name, "")) + else: + return "" diff --git a/conans/client/client_cache.py b/conans/client/client_cache.py index cb71da1f1f1..332416cb480 100644 --- a/conans/client/client_cache.py +++ b/conans/client/client_cache.py @@ -2,7 +2,7 @@ from collections import OrderedDict from conans.client.conf import ConanClientConfigParser, default_client_conf, default_settings_yml -from conans.client.detect import detect_defaults_settings +from conans.client.conf.detect import detect_defaults_settings from conans.client.output import Color from conans.client.profile_loader import read_profile from conans.errors import ConanException diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index dc37b0d9b4c..b7c3785f9ba 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -8,7 +8,7 @@ from conans import __version__ as client_version, tools from conans.client.client_cache import ClientCache from conans.client.conf import MIN_SERVER_COMPATIBLE_VERSION, ConanClientConfigParser -from conans.client.detect import detect_defaults_settings +from conans.client.conf.detect import detect_defaults_settings from conans.client.manager import ConanManager, existing_info_files from conans.client.migrations import ClientMigrator from conans.client.output import ConanOutput, ScopedOutput diff --git a/conans/client/detect.py b/conans/client/conf/detect.py similarity index 100% rename from conans/client/detect.py rename to conans/client/conf/detect.py diff --git a/conans/client/generators/gcc.py b/conans/client/generators/gcc.py index f65532ea00c..3bf43b86944 100644 --- a/conans/client/generators/gcc.py +++ b/conans/client/generators/gcc.py @@ -1,4 +1,4 @@ -from conans.client.configure_build_environment import architecture_dict, stdlib_flags, \ +from conans.client.build.autotools_environment import architecture_dict, stdlib_flags, \ stdlib_defines from conans.model import Generator from conans.paths import BUILD_INFO_GCC diff --git a/conans/client/generators/virtualbuildenv.py b/conans/client/generators/virtualbuildenv.py index 4809d0a5927..64f7353859d 100644 --- a/conans/client/generators/virtualbuildenv.py +++ b/conans/client/generators/virtualbuildenv.py @@ -1,5 +1,6 @@ -from conans.client.configure_build_environment import (AutoToolsBuildEnvironment, VisualStudioBuildEnvironment) from conans.client.generators.virtualenv import VirtualEnvGenerator +from conans.client.build.autotools_environment import AutoToolsBuildEnvironment +from conans.client.build.visual_environment import VisualStudioBuildEnvironment class VirtualBuildEnvGenerator(VirtualEnvGenerator): diff --git a/conans/client/manager.py b/conans/client/manager.py index d4984a94292..85aab3c7430 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -3,12 +3,10 @@ import time from collections import OrderedDict, Counter -import copy - from conans.client import packager from conans.client.client_cache import ClientCache from conans.client.deps_builder import DepsGraphBuilder -from conans.client.detect import detected_os +from conans.client.conf.detect import detected_os from conans.client.export import export_conanfile, load_export_conanfile from conans.client.generators import write_generators from conans.client.generators.text import TXTGenerator diff --git a/conans/test/command/install_test.py b/conans/test/command/install_test.py index dbdb81a0f5d..cae36aa64e9 100644 --- a/conans/test/command/install_test.py +++ b/conans/test/command/install_test.py @@ -8,7 +8,7 @@ from conans.model.info import ConanInfo from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.paths import CONANFILE_TXT -from conans.client.detect import detected_os +from conans.client.conf.detect import detected_os from conans.util.files import load diff --git a/conans/test/functional/autotools_configure_test.py b/conans/test/functional/autotools_configure_test.py index b2d170fca51..782d9ac4fc9 100644 --- a/conans/test/functional/autotools_configure_test.py +++ b/conans/test/functional/autotools_configure_test.py @@ -1,7 +1,7 @@ import platform import unittest -from conans.client.configure_build_environment import AutoToolsBuildEnvironment +from conans.client.build.autotools_environment import AutoToolsBuildEnvironment from conans import tools from conans.client.tools.oss import cpu_count from conans.test.utils.conanfile import MockConanfile, MockSettings diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 861e549906f..50a81b89c67 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -10,7 +10,7 @@ from conans.model.conan_file import ConanFile from conans.model.settings import Settings from conans.client.conf import default_settings_yml -from conans.client.cmake import CMake +from conans.client.build.cmake import CMake from conans.test.utils.tools import TestBufferConanOutput from conans.tools import cpu_count from conans.util.files import save diff --git a/conans/test/functional/compile_helpers_test.py b/conans/test/functional/compile_helpers_test.py index aec4de0a092..9809fd378a9 100644 --- a/conans/test/functional/compile_helpers_test.py +++ b/conans/test/functional/compile_helpers_test.py @@ -2,8 +2,8 @@ import platform import unittest -from conans.client.configure_environment import ConfigureEnvironment -from conans.client.gcc import GCC +from conans.client.build.configure_environment import ConfigureEnvironment +from conans.client.build.gcc import GCC from conans.client.runner import ConanRunner from conans.errors import ConanException from conans.model.env_info import DepsEnvInfo diff --git a/conans/test/integration/cmake_multi_test.py b/conans/test/integration/cmake_multi_test.py index a85668f4660..28f419833e2 100644 --- a/conans/test/integration/cmake_multi_test.py +++ b/conans/test/integration/cmake_multi_test.py @@ -4,7 +4,7 @@ import platform import os from conans.test.utils.multi_config import multi_config_files -from conans.client.cmake import clean_sh_from_path +from conans.client.build.cmake import clean_sh_from_path conanfile_py = """ from conans import ConanFile, CMake diff --git a/conans/test/integration/conf_default_settings_test.py b/conans/test/integration/conf_default_settings_test.py index 2bc8c38009f..ed10f95d175 100644 --- a/conans/test/integration/conf_default_settings_test.py +++ b/conans/test/integration/conf_default_settings_test.py @@ -5,7 +5,7 @@ from conans import tools from conans.client.client_cache import ClientCache -from conans.client.detect import detect_defaults_settings +from conans.client.conf.detect import detect_defaults_settings from conans.paths import CONANFILE_TXT from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient diff --git a/conans/test/util/detect_test.py b/conans/test/util/detect_test.py index c1a9bed0ef6..3ce7dc08a82 100644 --- a/conans/test/util/detect_test.py +++ b/conans/test/util/detect_test.py @@ -1,6 +1,6 @@ import unittest from conans.test.utils.tools import TestBufferConanOutput -from conans.client.detect import detect_defaults_settings +from conans.client.conf.detect import detect_defaults_settings import platform From 89d15316f9083a430e1d8adc7bb0f9c0f8c42506 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Thu, 2 Nov 2017 11:39:33 +0100 Subject: [PATCH 05/38] Not repeated arguments (#1978) --- conans/client/command.py | 148 +++++++++++++++++------------ conans/test/command/source_test.py | 18 +++- 2 files changed, 106 insertions(+), 60 deletions(-) diff --git a/conans/client/command.py b/conans/client/command.py index b4a2bda7f49..fbd9ede118b 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -42,6 +42,16 @@ def __call__(self, parser, namespace, values, option_strings=None): # @UnusedVa dest.append(values) +class OnceArgument(argparse.Action): + """Allows to declare a parameter that can have only one value, by default argparse takes the + latest declared and it's very confusing""" + def __call__(self, parser, namespace, values, option_string=None): + if getattr(namespace, self.dest) is not None and self.default is None: + msg = '{o} can only be specified once'.format(o=option_string) + raise argparse.ArgumentError(None, msg) + setattr(namespace, self.dest, values) + + class Command(object): """ A single command of the conan application, with all the first level commands. Manages the parsing of parameters and delegates functionality in @@ -211,14 +221,14 @@ def create(self, *args): parser.add_argument("reference", help='user/channel, or a full package reference' ' (Pkg/version@user/channel), if name and version ' ' are not declared in the recipe') - parser.add_argument('--cwd', '-c', default=None, + parser.add_argument('--cwd', '-c', default=None, action=OnceArgument, help='Optional. Folder with a %s. Default current directory.' % CONANFILE) - parser.add_argument("--file", "-f", help="specify conanfile filename") + parser.add_argument("--file", "-f", help="specify conanfile filename", action=OnceArgument) parser.add_argument("-ne", "--not-export", default=False, action='store_true', help='Do not export the conanfile') - parser.add_argument("-tf", "--test-folder", "--test_folder", + parser.add_argument("-tf", "--test-folder", "--test_folder", action=OnceArgument, help='alternative test folder name, by default is "test_package"') parser.add_argument('--keep-source', '-k', default=False, action='store_true', help='Optional. Do not remove the source folder in local cache. ' @@ -254,7 +264,8 @@ def download(self, *args): help='package recipe reference e.g., MyPackage/1.2@user/channel') parser.add_argument("--package", "-p", nargs=1, action=Extender, help='Force install specified package ID (ignore settings/options)') - parser.add_argument("-r", "--remote", help='look in the specified remote server') + parser.add_argument("-r", "--remote", help='look in the specified remote server', + action=OnceArgument) args = parser.parse_args(*args) reference = ConanFileReference.loads(args.reference) @@ -277,12 +288,12 @@ def install(self, *args): parser = argparse.ArgumentParser(description=self.install.__doc__, prog="conan install") parser.add_argument("path", nargs='?', default="", help='path to a recipe (conanfile.py). e.g., ./my_project/') - parser.add_argument("--file", "-f", help="specify conanfile filename") + parser.add_argument("--file", "-f", help="specify conanfile filename", action=OnceArgument) parser.add_argument("--generator", "-g", nargs=1, action=Extender, help='Generators to use') parser.add_argument("--werror", action='store_true', default=False, help='Error instead of warnings for graph inconsistencies') - parser.add_argument("--install-folder", "--install_folder", "-if", + parser.add_argument("--install-folder", "--install_folder", "-if", action=OnceArgument, help='Use this directory as the directory where to put the generator' 'files, conaninfo/conanbuildinfo.txt etc.') @@ -368,7 +379,7 @@ def info(self, *args): parser.add_argument("reference", nargs='?', default="", help='reference name or path to conanfile file, ' 'e.g., MyPackage/1.2@user/channel or ./my_project/') - parser.add_argument("--file", "-f", help="specify conanfile filename") + parser.add_argument("--file", "-f", help="specify conanfile filename", action=OnceArgument) parser.add_argument("--only", "-n", nargs=1, action=Extender, help='show the specified fields only from: ' '%s or use --paths with options %s. Use --only None to show only ' @@ -385,7 +396,7 @@ def info(self, *args): parser.add_argument("--json", "-j", nargs='?', const="1", type=str, help='Only with --build_order option, return the information in a json.' ' e.j --json=/path/to/filename.json or --json to output the json') - parser.add_argument("--graph", "-g", + parser.add_argument("--graph", "-g", action=OnceArgument, help='Creates file with project dependencies graph. It will generate ' 'a DOT or HTML file depending on the filename extension') build_help = 'given a build policy (same install command "build" parameter), return an ' \ @@ -454,9 +465,9 @@ def source(self, *args): parser = argparse.ArgumentParser(description=self.source.__doc__, prog="conan source") parser.add_argument("path", help='path to a recipe (conanfile.py), e.g., conan source .') - parser.add_argument("--source-folder", "--source_folder", "-s", + parser.add_argument("--source-folder", "--source_folder", "-s", action=OnceArgument, help='Destination directory. Defaulted to current directory') - parser.add_argument("--install-folder", "-if", + parser.add_argument("--install-folder", "-if", action=OnceArgument, help="Optional. Local folder containing the conaninfo.txt and " "conanbuildinfo.txt " "files (from a previous conan install execution). Defaulted to the " @@ -478,6 +489,7 @@ def source(self, *args): except ConanException: pass + print(args.source_folder) return self._conan.source(args.path, args.source_folder, args.install_folder) def build(self, *args): @@ -489,21 +501,21 @@ def build(self, *args): parser = argparse.ArgumentParser(description=self.build.__doc__, prog="conan build") parser.add_argument("path", help='path to a recipe (conanfile.py), e.g., conan build .') - parser.add_argument("--file", "-f", help="specify conanfile filename") - parser.add_argument("--source-folder", "--source_folder", "-sf", + parser.add_argument("--file", "-f", help="specify conanfile filename", action=OnceArgument) + parser.add_argument("--source-folder", "--source_folder", "-sf", action=OnceArgument, help="local folder containing the sources. Defaulted to the directory " "of the conanfile. A relative path can also be specified " "(relative to the current directory)") - parser.add_argument("--build-folder", "--build_folder", "-bf", + parser.add_argument("--build-folder", "--build_folder", "-bf", action=OnceArgument, help="build folder, working directory of the build process. Defaulted " "to the current directory. A relative path can also be specified " "(relative to the current directory)") - parser.add_argument("--package-folder", "--package_folder", "-pf", + parser.add_argument("--package-folder", "--package_folder", "-pf", action=OnceArgument, help="folder to install the package (when the build system or build() " "method does it). Defaulted to the '{build_folder}/package' folder" ". A relative path can be specified, relative to the current " " folder. Also an absolute path is allowed.") - parser.add_argument("--install-folder", "--install_folder", "-if", + parser.add_argument("--install-folder", "--install_folder", "-if", action=OnceArgument, help="Optional. Local folder containing the conaninfo.txt and " "conanbuildinfo.txt files (from a previous conan install " "execution). Defaulted to --build-folder") @@ -524,20 +536,20 @@ def package(self, *args): """ parser = argparse.ArgumentParser(description=self.package.__doc__, prog="conan package") parser.add_argument("path", help='path to a recipe (conanfile.py), e.g., conan package .') - parser.add_argument("--source-folder", "--source_folder", "-sf", + parser.add_argument("--source-folder", "--source_folder", "-sf", action=OnceArgument, help="local folder containing the sources. Defaulted to the directory " "of the conanfile. A relative path can also be specified " "(relative to the current directory)") - parser.add_argument("--build-folder", "--build_folder", "-bf", + parser.add_argument("--build-folder", "--build_folder", "-bf", action=OnceArgument, help="build folder, working directory of the build process. Defaulted " "to the current directory. A relative path can also be specified " "(relative to the current directory)") - parser.add_argument("--package-folder", "--package_folder", "-pf", + parser.add_argument("--package-folder", "--package_folder", "-pf", action=OnceArgument, help="folder to install the package. Defaulted to the " "'{build_folder}/package' folder. A relative path can be specified" " (relative to the current directory). Also an absolute path" " is allowed.") - parser.add_argument("--install-folder", "--install_folder", "-if", + parser.add_argument("--install-folder", "--install_folder", "-if", action=OnceArgument, help="Optional. Local folder containing the conaninfo.txt and " "conanbuildinfo.txt files (from a previous conan install " "execution). Defaulted to --build-folder ") @@ -573,11 +585,11 @@ def imports(self, *args): "containing the conan_imports_manifest.txt file generated in a previous" "execution. e.j: conan imports ./imported_files --undo ") parser.add_argument("--file", "-f", help="Use another filename, " - "e.g.: conan imports -f=conanfile2.py") - parser.add_argument("--import-folder", "--import_folder", "-imf", + "e.g.: conan imports -f=conanfile2.py", action=OnceArgument) + parser.add_argument("--import-folder", "--import_folder", "-imf", action=OnceArgument, help="Directory to copy the artifacts to. By default it will be the" " current directory") - parser.add_argument("--install-folder", "--install_folder", "-if", + parser.add_argument("--install-folder", "--install_folder", "-if", action=OnceArgument, help="local folder containing the conaninfo.txt and conanbuildinfo.txt " "files (from a previous conan install execution)") parser.add_argument("-u", "--undo", default=False, action="store_true", @@ -609,20 +621,20 @@ def export_pkg(self, *args): parser.add_argument("reference", help='user/channel, or a full package reference' ' (Pkg/version@user/channel), if name and version ' ' are not declared in the recipe (conanfile.py)') - parser.add_argument("--source-folder", "--source_folder", "-sf", + parser.add_argument("--source-folder", "--source_folder", "-sf", action=OnceArgument, help="local folder containing the sources. Defaulted to --build-folder." " A relative path to the current dir can also be specified" ) - parser.add_argument("--build-folder", "--build_folder", "-bf", + parser.add_argument("--build-folder", "--build_folder", "-bf", action=OnceArgument, help="build folder, working directory of the build process. Defaulted " "to the current directory. A relative path can also be specified " "(relative to the current directory)") - parser.add_argument("--install-folder", "-if", + parser.add_argument("--install-folder", "-if", action=OnceArgument, help="local folder containing the conaninfo.txt and conanbuildinfo.txt " "files (from a previous conan install execution). Defaulted to " "--build-folder. If these files are found in the specified folder, " "they will be used, then if you specify --profile, -s, -o, --env, " "it will raise an error.") - parser.add_argument("--profile", "-pr", + parser.add_argument("--profile", "-pr", action=OnceArgument, help='Profile for this package') parser.add_argument("--options", "-o", help='Define options values, e.g., -o Pkg:with_qt=true', @@ -664,10 +676,10 @@ def export(self, *args): parser.add_argument("reference", help='user/channel, or a full package reference' ' (Pkg/version@user/channel), if name and version ' ' are not declared in the recipe') - parser.add_argument('--path', '-p', default=None, + parser.add_argument('--path', '-p', default=None, action=OnceArgument, help='Optional. Folder with a %s. Default current directory.' % CONANFILE) - parser.add_argument("--file", "-f", help="specify conanfile filename") + parser.add_argument("--file", "-f", help="specify conanfile filename", action=OnceArgument) parser.add_argument('--keep-source', '-k', default=False, action='store_true', help='Optional. Do not remove the source folder in the local cache. ' 'Use for testing purposes only') @@ -698,13 +710,15 @@ def remove(self, *args): help='Remove source folders') parser.add_argument('-f', '--force', default=False, action='store_true', help='Remove without requesting a confirmation') - parser.add_argument('-r', '--remote', help='Will remove from the specified remote') - parser.add_argument('-q', '--query', default=None, help='Packages query: "os=Windows AND ' - '(arch=x86 OR compiler=gcc)".' - ' The "pattern" parameter ' - 'has to be a package recipe ' - 'reference: MyPackage/1.2' - '@user/channel') + parser.add_argument('-r', '--remote', help='Will remove from the specified remote', + action=OnceArgument) + parser.add_argument('-q', '--query', default=None, action=OnceArgument, + help='Packages query: "os=Windows AND ' + '(arch=x86 OR compiler=gcc)".' + ' The "pattern" parameter ' + 'has to be a package recipe ' + 'reference: MyPackage/1.2' + '@user/channel') parser.add_argument("--outdated", "-o", help="Remove only outdated from recipe packages", default=False, action="store_true") args = parser.parse_args(*args) @@ -756,12 +770,15 @@ def user(self, *parameters): parser.add_argument("name", nargs='?', default=None, help='Username you want to use. ' 'If no name is provided it will show the current user.') - parser.add_argument("--remote", "-r", help='look in the specified remote server') + parser.add_argument("--remote", "-r", help='look in the specified remote server', + action=OnceArgument) parser.add_argument('-c', '--clean', default=False, action='store_true', help='Remove user and tokens for all remotes') parser.add_argument("-p", "--password", nargs='?', const="", type=str, - help='User password. Use double quotes if password with spacing, and escape quotes if ' - 'existing. If empty, the password is requested interactively (not exposed)') + action=OnceArgument, + help='User password. Use double quotes if password with spacing, ' + 'and escape quotes if existing. If empty, the password is ' + 'requested interactively (not exposed)') args = parser.parse_args(*parameters) # To enable -h return self._conan.user(name=args.name, clean=args.clean, remote=args.remote, password=args.password) @@ -785,18 +802,20 @@ def search(self, *args): action='store_true', help='Make a case-sensitive search. ' 'Use it to guarantee case-sensitive ' 'search in Windows or other case-insensitive filesystems') - parser.add_argument('-r', '--remote', help='Remote origin') + parser.add_argument('-r', '--remote', help='Remote origin', action=OnceArgument) parser.add_argument('--raw', default=False, action='store_true', help='Print just the list of recipes') - parser.add_argument('--table', + parser.add_argument('--table', action=OnceArgument, help='Outputs html file with a table of binaries. Only valid if ' '"pattern" is a package recipe reference') - parser.add_argument('-q', '--query', default=None, help='Packages query: "os=Windows AND ' - '(arch=x86 OR compiler=gcc)".' - ' The "pattern" parameter ' - 'has to be a package recipe ' - 'reference: MyPackage/1.2' - '@user/channel') + parser.add_argument('-q', '--query', default=None, action=OnceArgument, + help='Packages query: "os=Windows AND ' + '(arch=x86 OR compiler=gcc)".' + ' The "pattern" parameter ' + 'has to be a package recipe ' + 'reference: MyPackage/1.2' + '@user/channel'), + parser.add_argument('-o', '--outdated', default=False, action='store_true', help='Show only outdated from recipe packages') args = parser.parse_args(*args) @@ -844,8 +863,10 @@ def upload(self, *args): parser.add_argument('pattern', help='Pattern or package recipe reference, ' 'e.g., "openssl/*", "MyPackage/1.2@user/channel"') # TODO: packageparser.add_argument('package', help='user name') - parser.add_argument("--package", "-p", default=None, help='package ID to upload') - parser.add_argument("--remote", "-r", help='upload to this specific remote') + parser.add_argument("--package", "-p", default=None, help='package ID to upload', + action=OnceArgument) + parser.add_argument("--remote", "-r", help='upload to this specific remote', + action=OnceArgument) parser.add_argument("--all", action='store_true', default=False, help='Upload both package recipe and packages') parser.add_argument("--skip-upload", "--skip_upload", action='store_true', @@ -862,9 +883,11 @@ def upload(self, *args): help='If pattern is given upload all matching recipes without ' 'confirmation') parser.add_argument('--retry', default=2, type=int, - help='In case of fail retries to upload again the specified times') + help='In case of fail retries to upload again the specified times', + action=OnceArgument) parser.add_argument('--retry-wait', '--retry_wait', default=5, type=int, - help='Waits specified seconds before retry again') + help='Waits specified seconds before retry again', + action=OnceArgument) args = parser.parse_args(*args) return self._conan.upload(pattern=args.pattern, package=args.package, remote=args.remote, @@ -888,7 +911,7 @@ def remote(self, *args): help='Verify SSL certificated. Default True', default="True", nargs="?") parser_add.add_argument("-i", "--insert", nargs="?", const=0, type=int, - help="insert remote at specific index") + help="insert remote at specific index", action=OnceArgument) parser_rm = subparsers.add_parser('remove', help='remove a remote') parser_rm.add_argument('remote', help='name of the remote') parser_upd = subparsers.add_parser('update', help='update the remote url') @@ -898,7 +921,8 @@ def remote(self, *args): help='Verify SSL certificated. Default True', default="True", nargs="?") parser_upd.add_argument("-i", "--insert", nargs="?", const=0, type=int, - help="insert remote at specific index") + help="insert remote at specific index", + action=OnceArgument) subparsers.add_parser('list_ref', help='list the package recipes and its associated remotes') @@ -1012,8 +1036,10 @@ def get(self, *args): 'conafile if only a reference is specified and a conaninfo.txt ' 'file contents if the package is also specified', default=None, nargs="?") - parser.add_argument("--package", "-p", default=None, help='package ID') - parser.add_argument("--remote", "-r", help='Get from this specific remote') + parser.add_argument("--package", "-p", default=None, help='package ID', + action=OnceArgument) + parser.add_argument("--remote", "-r", help='Get from this specific remote', + action=OnceArgument) parser.add_argument("--raw", "-raw", help='Do not decorate the text', default=False, action='store_true') args = parser.parse_args(*args) @@ -1172,14 +1198,17 @@ def get_reference_fields(arg_reference): def _add_manifests_arguments(parser): parser.add_argument("--manifests", "-m", const=default_manifest_folder, nargs="?", help='Install dependencies manifests in folder for later verify.' - ' Default folder is .conan_manifests, but can be changed') + ' Default folder is .conan_manifests, but can be changed', + action=OnceArgument) parser.add_argument("--manifests-interactive", "-mi", const=default_manifest_folder, nargs="?", help='Install dependencies manifests in folder for later verify, ' 'asking user for confirmation. ' - 'Default folder is .conan_manifests, but can be changed') + 'Default folder is .conan_manifests, but can be changed', + action=OnceArgument) parser.add_argument("--verify", "-v", const=default_manifest_folder, nargs="?", - help='Verify dependencies manifests against stored ones') + help='Verify dependencies manifests against stored ones', + action=OnceArgument) def _add_common_install_arguments(parser, build_help): @@ -1187,9 +1216,10 @@ def _add_common_install_arguments(parser, build_help): help="check updates exist from upstream remotes") parser.add_argument("--scope", "-sc", nargs=1, action=Extender, help='Use the specified scope in the install command') - parser.add_argument("--profile", "-pr", default=None, + parser.add_argument("--profile", "-pr", default=None, action=OnceArgument, help='Apply the specified profile to the install command') - parser.add_argument("-r", "--remote", help='look in the specified remote server') + parser.add_argument("-r", "--remote", help='look in the specified remote server', + action=OnceArgument) parser.add_argument("--options", "-o", help='Define options values, e.g., -o Pkg:with_qt=true', nargs=1, action=Extender) diff --git a/conans/test/command/source_test.py b/conans/test/command/source_test.py index ad51f14ff74..56a351fbe96 100644 --- a/conans/test/command/source_test.py +++ b/conans/test/command/source_test.py @@ -1,5 +1,5 @@ import unittest - +from conans.errors import ConanException from conans.paths import CONANFILE, BUILD_INFO from conans.test.utils.tools import TestClient from conans.util.files import load, mkdir @@ -126,6 +126,22 @@ def source(self): self.assertIn("OTHERVAR=bar", client.out) self.assertIn("CURDIR=%s" % src_folder, client.out) + def repeat_args_fails_test(self): + conanfile = ''' +from conans import ConanFile +class ConanLib(ConanFile): + + def source(self): + pass +''' + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("source . --source-folder sf") + with self.assertRaisesRegexp(Exception, "Command failed"): + client.run("source . --source-folder sf --source-folder sf") + with self.assertRaisesRegexp(Exception, "Command failed"): + client.run("source . --source-folder sf --install-folder if --install-folder rr") + def local_source_test(self): conanfile = ''' from conans import ConanFile From 00060ab946b122f5c909a2b7674bbebda0198f9d Mon Sep 17 00:00:00 2001 From: James Date: Wed, 8 Nov 2017 12:51:18 +0100 Subject: [PATCH 06/38] more version ranges tests (#1977) * more version ranges tests * new test --- conans/test/model/version_ranges_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conans/test/model/version_ranges_test.py b/conans/test/model/version_ranges_test.py index 71e4ac449a5..b71c0777a37 100644 --- a/conans/test/model/version_ranges_test.py +++ b/conans/test/model/version_ranges_test.py @@ -26,6 +26,12 @@ def prereleases_versions_test(self): self.assertEqual(result, "1.1.1-a.111") result = satisfying(["1.1.1", "1.1.1-11", "1.1.1-111", "1.1.1-21"], "", output) self.assertEqual(result, "1.1.1") + result = satisfying(["4.2.2", "4.2.3-pre"], "~4.2.3-", output) + self.assertEqual(result, "4.2.3-pre") + result = satisfying(["4.2.2", "4.2.3-pre", "4.2.4"], "~4.2.3-", output) + self.assertEqual(result, "4.2.4") + result = satisfying(["4.2.2", "4.2.3-pre", "4.2.3"], "~4.2.3-", output) + self.assertEqual(result, "4.2.3") def basic_test(self): output = TestBufferConanOutput() From c7ee48add5645d39593002effca3682336400891 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 8 Nov 2017 12:52:23 +0100 Subject: [PATCH 07/38] do not create conaninfo for pkg-ref install (#1980) --- conans/client/manager.py | 9 +++++---- conans/test/command/install_cwd_test.py | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/conans/client/manager.py b/conans/client/manager.py index 85aab3c7430..839968f79da 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -409,10 +409,11 @@ def install(self, reference, install_folder, profile, remote=None, tmp.extend([g for g in generators if g not in tmp]) conanfile.generators = tmp write_generators(conanfile, install_folder, output) - # Write conaninfo - content = normalize(conanfile.info.dumps()) - save(os.path.join(install_folder, CONANINFO), content) - output.info("Generated %s" % CONANINFO) + if not isinstance(reference, ConanFileReference): + # Write conaninfo + content = normalize(conanfile.info.dumps()) + save(os.path.join(install_folder, CONANINFO), content) + output.info("Generated %s" % CONANINFO) if not no_imports: run_imports(conanfile, install_folder, output) call_system_requirements(conanfile, output) diff --git a/conans/test/command/install_cwd_test.py b/conans/test/command/install_cwd_test.py index 9a48ade5269..cdf5cffde44 100644 --- a/conans/test/command/install_cwd_test.py +++ b/conans/test/command/install_cwd_test.py @@ -18,6 +18,7 @@ class MyLib(ConanFile): version = "0.1" exports = "*" settings = "os", "compiler", "arch", "build_type" + generators = "cmake" def build(self): pass @@ -30,3 +31,7 @@ def test_install_cwd(self): os.mkdir(os.path.join(self.client.current_folder, "new")) self.client.run("install . --install-folder new -g cmake") self.assertTrue(os.path.join(self.client.current_folder, "new", "conanbuildinfo.cmake")) + + def install_ref(self): + self.client.run("install MyLib/0.1@lasote/stable --build=missing") + self.assertEqual(["conanfile.py"], os.listdir(self.client.current_folder)) From 6d50435c169d85803fbfd2702af141ecf7c0d6a5 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 8 Nov 2017 12:54:58 +0100 Subject: [PATCH 08/38] adding support for checking existing option with in (#1976) --- conans/model/options.py | 6 ++++++ conans/test/model/options_test.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/conans/model/options.py b/conans/model/options.py index 48001896070..e61326f4584 100644 --- a/conans/model/options.py +++ b/conans/model/options.py @@ -345,6 +345,9 @@ def __init__(self, definition): for k, v in definition.items()} self._modified = {} + def __contains__(self, option): + return str(option) in self._data + @staticmethod def loads(text): return PackageOptions(yaml.load(text) or {}) @@ -464,6 +467,9 @@ def deps_package_values(self): def clear(self): self._package_options.clear() + def __contains__(self, option): + return option in self._package_options + def __getitem__(self, item): return self._deps_package_values.setdefault(item, PackageOptionValues()) diff --git a/conans/test/model/options_test.py b/conans/test/model/options_test.py index 79773095459..98348261483 100644 --- a/conans/test/model/options_test.py +++ b/conans/test/model/options_test.py @@ -19,6 +19,14 @@ def setUp(self): package_options.values = values self.sut = Options(package_options) + def test_in(self): + package_options = PackageOptions.loads("{static: [True, False]}") + sut = Options(package_options) + self.assertTrue("static" in sut) + self.assertFalse("shared" in sut) + self.assertTrue("shared" not in sut) + self.assertFalse("static" not in sut) + def undefined_value_test(self): """ Not assigning a value to options will raise an error at validate() step """ From fd757238811b390b017e1bf26bb0e07a1e1ffddd Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 09:41:08 +0100 Subject: [PATCH 09/38] don't rebuild conan created pkg because build_policy=always (#1974) --- conans/client/manager.py | 8 +++++--- conans/test/command/create_test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/conans/client/manager.py b/conans/client/manager.py index 839968f79da..e109b6c6b83 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -43,6 +43,7 @@ def __init__(self, params, output): self._out = output self.outdated = False self.missing = False + self.never = False self.patterns = [] self._unused_patterns = [] self.all = False @@ -53,22 +54,23 @@ def __init__(self, params, output): if len(params) == 0: self.all = True else: - never = False for param in params: if param == "outdated": self.outdated = True elif param == "missing": self.missing = True elif param == "never": - never = True + self.never = True else: self.patterns.append("%s" % param) - if never and (self.outdated or self.missing or self.patterns): + if self.never and (self.outdated or self.missing or self.patterns): raise ConanException("--build=never not compatible with other options") self._unused_patterns = list(self.patterns) def forced(self, conan_file, reference): + if self.never: + return False if self.all: return True diff --git a/conans/test/command/create_test.py b/conans/test/command/create_test.py index 6d5a0d8b0d8..4949e08af21 100644 --- a/conans/test/command/create_test.py +++ b/conans/test/command/create_test.py @@ -261,3 +261,33 @@ def test(self): # Test that the build require is applyed to testing client.run("create conan/stable --profile ./myprofile --build missing") self.assertIn("TESTING!!", client.user_io.out) + + def build_policy_test(self): + # https://github.com/conan-io/conan/issues/1956 + client = TestClient() + conanfile = ''' +from conans import ConanFile + +class HelloConan(ConanFile): + name = "HelloBar" + version = "0.1" + build_policy = "always" +''' + test_package = ''' +from conans import ConanFile + +class HelloTestConan(ConanFile): + requires = "HelloBar/0.1@lasote/testing" + def test(self): + pass +''' + client.save({"conanfile.py": conanfile, "test_package/conanfile.py": test_package}) + client.run("create lasote/testing") + self.assertIn("HelloBar/0.1@lasote/testing: WARN: Forced build from source", + client.out) + client.save({"conanfile.py": conanfile.replace("HelloBar", "Hello") + + " requires='HelloBar/0.1@lasote/testing'", + "test_package/conanfile.py": test_package.replace("HelloBar", "Hello")}) + client.run("create lasote/stable") + self.assertIn("HelloBar/0.1@lasote/testing: WARN: Forced build from source", + client.out) From da78c52a4797dd165c59815b1e6ebf116f8ccf4d Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 09:51:53 +0100 Subject: [PATCH 10/38] some cmd refactor (#1996) --- conans/client/cmd/__init__.py | 0 conans/client/cmd/copy.py | 75 +++++++++++++++++++ conans/client/{ => cmd}/export.py | 47 ++++++++++-- .../{linter.py => cmd/export_linter.py} | 0 conans/client/{ => cmd}/new.py | 12 +-- conans/client/{ => cmd}/new_ci.py | 0 conans/client/command.py | 5 +- conans/client/conan_api.py | 37 ++++----- conans/client/manager.py | 60 +-------------- conans/client/package_copier.py | 52 ------------- conans/test/functional/package_copier_test.py | 22 +++--- conans/test/utils/tools.py | 2 +- 12 files changed, 163 insertions(+), 149 deletions(-) create mode 100644 conans/client/cmd/__init__.py create mode 100644 conans/client/cmd/copy.py rename conans/client/{ => cmd}/export.py (74%) rename conans/client/{linter.py => cmd/export_linter.py} (100%) rename conans/client/{ => cmd}/new.py (95%) rename conans/client/{ => cmd}/new_ci.py (100%) delete mode 100644 conans/client/package_copier.py diff --git a/conans/client/cmd/__init__.py b/conans/client/cmd/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/client/cmd/copy.py b/conans/client/cmd/copy.py new file mode 100644 index 00000000000..3f201e64dc2 --- /dev/null +++ b/conans/client/cmd/copy.py @@ -0,0 +1,75 @@ +from conans.model.ref import PackageReference, ConanFileReference +import os +from conans.util.files import rmdir +import shutil +from conans.errors import ConanException +from conans.client.loader_parse import load_conanfile_class +from conans.client.proxy import ConanProxy + + +def _prepare_sources(client_cache, user_io, remote_manager, reference): + remote_proxy = ConanProxy(client_cache, user_io, remote_manager, None) + conan_file_path = client_cache.conanfile(reference) + conanfile = load_conanfile_class(conan_file_path) + remote_proxy.complete_recipe_sources(reference, short_paths=conanfile.short_paths) + return conanfile.short_paths + + +def _get_package_ids(client_cache, reference, package_ids): + if not package_ids or package_ids is True: + packages = client_cache.packages(reference) + if os.path.exists(packages): + package_ids = os.listdir(packages) + else: + package_ids = [] + return package_ids + + +def cmd_copy(reference, user_channel, package_ids, client_cache, user_io, remote_manager, + force=False): + src_ref = ConanFileReference.loads(reference) + + short_paths = _prepare_sources(client_cache, user_io, remote_manager, src_ref) + package_ids = _get_package_ids(client_cache, src_ref, package_ids) + package_copy(src_ref, user_channel, package_ids, client_cache, user_io, + short_paths, force) + + +def package_copy(src_ref, user_channel, package_ids, paths, user_io, + short_paths=False, force=False): + dest_ref = ConanFileReference.loads("%s/%s@%s" % (src_ref.name, + src_ref.version, + user_channel)) + # Copy export + export_origin = paths.export(src_ref) + if not os.path.exists(export_origin): + raise ConanException("'%s' doesn't exist" % str(src_ref)) + export_dest = paths.export(dest_ref) + if os.path.exists(export_dest): + if not force and not user_io.request_boolean("'%s' already exist. Override?" + % str(dest_ref)): + return + rmdir(export_dest) + shutil.copytree(export_origin, export_dest, symlinks=True) + user_io.out.info("Copied %s to %s" % (str(src_ref), str(dest_ref))) + + export_sources_origin = paths.export_sources(src_ref, short_paths) + export_sources_dest = paths.export_sources(dest_ref, short_paths) + if os.path.exists(export_sources_dest): + rmdir(export_sources_dest) + shutil.copytree(export_sources_origin, export_sources_dest, symlinks=True) + user_io.out.info("Copied sources %s to %s" % (str(src_ref), str(dest_ref))) + + # Copy packages + for package_id in package_ids: + package_origin = PackageReference(src_ref, package_id) + package_dest = PackageReference(dest_ref, package_id) + package_path_origin = paths.package(package_origin, short_paths) + package_path_dest = paths.package(package_dest, short_paths) + if os.path.exists(package_path_dest): + if not force and not user_io.request_boolean("Package '%s' already exist." + " Override?" % str(package_id)): + continue + rmdir(package_path_dest) + shutil.copytree(package_path_origin, package_path_dest, symlinks=True) + user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref))) diff --git a/conans/client/export.py b/conans/client/cmd/export.py similarity index 74% rename from conans/client/export.py rename to conans/client/cmd/export.py index 29f9e7db6d2..ad8f1f50f89 100644 --- a/conans/client/export.py +++ b/conans/client/cmd/export.py @@ -4,6 +4,8 @@ import shutil import os + +from conans.util.log import logger from conans.util.files import save, load, rmdir, is_dirty, set_dirty from conans.paths import CONAN_MANIFEST, CONANFILE from conans.errors import ConanException @@ -12,9 +14,44 @@ from conans.client.file_copier import FileCopier from conans.model.conan_file import create_exports, create_exports_sources from conans.client.loader_parse import load_conanfile_class - - -def load_export_conanfile(conanfile_path, output, name, version): +from conans.client.cmd.export_linter import conan_linter +from conans.model.ref import ConanFileReference + + +def cmd_export(user, channel, conan_file_path, output, search_manager, client_cache, + keep_source=False, filename=None, name=None, version=None): + """ Export the recipe + param conanfile_path: the original source directory of the user containing a + conanfile.py + param user: user under this package will be exported + param channel: string (stable, testing,...) + """ + assert conan_file_path + logger.debug("Exporting %s" % conan_file_path) + + src_folder = conan_file_path + conanfile_name = filename or CONANFILE + conan_file_path = os.path.join(conan_file_path, conanfile_name) + if ((os.path.exists(conan_file_path) and conanfile_name not in os.listdir(src_folder)) or + (conanfile_name != "conanfile.py" and conanfile_name.lower() == "conanfile.py")): + raise ConanException("Wrong '%s' case" % conanfile_name) + conan_linter(conan_file_path, output) + conanfile = _load_export_conanfile(conan_file_path, output, name, version) + conan_ref = ConanFileReference(conanfile.name, conanfile.version, user, channel) + conan_ref_str = str(conan_ref) + # Maybe a platform check could be added, but depends on disk partition + refs = search_manager.search(conan_ref_str, ignorecase=True) + if refs and conan_ref not in refs: + raise ConanException("Cannot export package with same name but different case\n" + "You exported '%s' but already existing '%s'" + % (conan_ref_str, " ".join(str(s) for s in refs))) + output = ScopedOutput(str(conan_ref), output) + with client_cache.conanfile_write_lock(conan_ref): + _export_conanfile(output, client_cache, conanfile, src_folder, conan_ref, keep_source, + filename) + + +def _load_export_conanfile(conanfile_path, output, name, version): conanfile = load_conanfile_class(conanfile_path) for field in ["url", "license", "description"]: @@ -53,7 +90,7 @@ def load_export_conanfile(conanfile_path, output, name, version): return conanfile -def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_source, filename): +def _export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_source, filename): destination_folder = paths.export(conan_ref) exports_source_folder = paths.export_sources(conan_ref, conanfile.short_paths) previous_digest = _init_export_folder(destination_folder, exports_source_folder) @@ -132,7 +169,7 @@ def classify_patterns(patterns): try: os.unlink(os.path.join(origin_folder, CONANFILE + 'c')) - except: + except OSError: pass copier = FileCopier(origin_folder, destination_folder) diff --git a/conans/client/linter.py b/conans/client/cmd/export_linter.py similarity index 100% rename from conans/client/linter.py rename to conans/client/cmd/export_linter.py diff --git a/conans/client/new.py b/conans/client/cmd/new.py similarity index 95% rename from conans/client/new.py rename to conans/client/cmd/new.py index 33f428a2756..94424d6bc8d 100644 --- a/conans/client/new.py +++ b/conans/client/cmd/new.py @@ -1,7 +1,7 @@ import re from conans.errors import ConanException from conans.model.ref import ConanFileReference -from conans.client.new_ci import ci_get_files +from conans.client.cmd.new_ci import ci_get_files conanfile = """from conans import ConanFile, CMake, tools @@ -35,7 +35,7 @@ def build(self): # Explicit way: # self.run('cmake %s/hello %s' % (self.source_folder, cmake.command_line)) # self.run("cmake --build . %s" % cmake.build_config) - + def package(self): self.copy("*.h", dst="include", src="hello") self.copy("*hello.lib", dst="lib", keep_path=False) @@ -213,9 +213,9 @@ def test(self): """ -def get_files(ref, header=False, pure_c=False, test=False, exports_sources=False, bare=False, - visual_versions=None, linux_gcc_versions=None, linux_clang_versions=None, osx_clang_versions=None, - shared=None, upload_url=None, gitignore=None, gitlab_gcc_versions=None, gitlab_clang_versions=None): +def cmd_new(ref, header=False, pure_c=False, test=False, exports_sources=False, bare=False, + visual_versions=None, linux_gcc_versions=None, linux_clang_versions=None, osx_clang_versions=None, + shared=None, upload_url=None, gitignore=None, gitlab_gcc_versions=None, gitlab_clang_versions=None): try: tokens = ref.split("@") name, version = tokens[0].split("/") @@ -226,7 +226,7 @@ def get_files(ref, header=False, pure_c=False, test=False, exports_sources=False pattern = re.compile('[\W_]+') package_name = pattern.sub('', name).capitalize() - except: + except ValueError: raise ConanException("Bad parameter, please use full package name," "e.g: MyLib/1.2.3@user/testing") diff --git a/conans/client/new_ci.py b/conans/client/cmd/new_ci.py similarity index 100% rename from conans/client/new_ci.py rename to conans/client/cmd/new_ci.py diff --git a/conans/client/command.py b/conans/client/command.py index fbd9ede118b..054acc24d04 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -755,9 +755,12 @@ def copy(self, *args): default=False, help='Override destination packages and the package recipe') args = parser.parse_args(*args) + if args.all and args.package: + raise ConanException("Cannot specify both --all and --package") + return self._conan.copy(reference=args.reference, user_channel=args.user_channel, force=args.force, - all=args.all, package=args.package) + packages=args.package or args.all) def user(self, *parameters): """ Authenticates against a remote with user/pass, caching the auth token. diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index b7c3785f9ba..ebea6242b67 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -41,6 +41,7 @@ from conans.tools import set_global_instances from conans.client.uploader import is_a_reference + default_manifest_folder = '.conan_manifests' @@ -159,17 +160,17 @@ def new(self, name, header=False, pure_c=False, test=False, exports_sources=Fals cwd=None, visual_versions=None, linux_gcc_versions=None, linux_clang_versions=None, osx_clang_versions=None, shared=None, upload_url=None, gitignore=None, gitlab_gcc_versions=None, gitlab_clang_versions=None): - from conans.client.new import get_files + from conans.client.cmd.new import cmd_new cwd = prepare_cwd(cwd) - files = get_files(name, header=header, pure_c=pure_c, test=test, - exports_sources=exports_sources, bare=bare, - visual_versions=visual_versions, - linux_gcc_versions=linux_gcc_versions, - linux_clang_versions=linux_clang_versions, - osx_clang_versions=osx_clang_versions, shared=shared, - upload_url=upload_url, gitignore=gitignore, - gitlab_gcc_versions=gitlab_gcc_versions, - gitlab_clang_versions=gitlab_clang_versions) + files = cmd_new(name, header=header, pure_c=pure_c, test=test, + exports_sources=exports_sources, bare=bare, + visual_versions=visual_versions, + linux_gcc_versions=linux_gcc_versions, + linux_clang_versions=linux_clang_versions, + osx_clang_versions=osx_clang_versions, shared=shared, + upload_url=upload_url, gitignore=gitignore, + gitlab_gcc_versions=gitlab_gcc_versions, + gitlab_clang_versions=gitlab_clang_versions) save_files(cwd, files) for f in sorted(files): @@ -683,14 +684,14 @@ def remove(self, pattern, query=None, packages=None, builds=None, src=False, for outdated=outdated) @api_method - def copy(self, reference="", user_channel="", force=False, all=False, package=None): - reference = ConanFileReference.loads(reference) - new_ref = ConanFileReference.loads("%s/%s@%s" % (reference.name, - reference.version, - user_channel)) - if all: - package = [] - self._manager.copy(reference, package, new_ref.user, new_ref.channel, force) + def copy(self, reference, user_channel, force=False, packages=None): + """ + param packages: None=No binaries, True=All binaries, else list of IDs + """ + from conans.client.cmd.copy import cmd_copy + # FIXME: conan copy does not support short-paths in Windows + cmd_copy(reference, user_channel, packages, self._client_cache, + self._user_io, self._manager._remote_manager, force=force) @api_method def user(self, name=None, clean=False, remote=None, password=None): diff --git a/conans/client/manager.py b/conans/client/manager.py index e109b6c6b83..77fd7e815ee 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -7,7 +7,6 @@ from conans.client.client_cache import ClientCache from conans.client.deps_builder import DepsGraphBuilder from conans.client.conf.detect import detected_os -from conans.client.export import export_conanfile, load_export_conanfile from conans.client.generators import write_generators from conans.client.generators.text import TXTGenerator from conans.client.importer import run_imports, undo_imports, run_deploy @@ -15,7 +14,6 @@ from conans.client.loader import ConanFileLoader from conans.client.manifest_manager import ManifestManager from conans.client.output import ScopedOutput, Color -from conans.client.package_copier import PackageCopier from conans.client.printer import Printer from conans.client.profile_loader import read_conaninfo_profile from conans.client.proxy import ConanProxy @@ -32,9 +30,7 @@ from conans.util.files import save, rmdir, normalize, mkdir, load from conans.util.log import logger from conans.model.manifest import FileTreeManifest -from conans.client.loader_parse import load_conanfile_class from conans.client.build_requires import BuildRequires -from conans.client.linter import conan_linter from conans.search.search import filter_outdated @@ -138,35 +134,9 @@ def get_loader(self, profile): def export(self, user, channel, conan_file_path, keep_source=False, filename=None, name=None, version=None): - """ Export the conans - param conanfile_path: the original source directory of the user containing a - conanfile.py - param user: user under this package will be exported - param channel: string (stable, testing,...) - """ - assert conan_file_path - logger.debug("Exporting %s" % conan_file_path) - - src_folder = conan_file_path - conanfile_name = filename or CONANFILE - conan_file_path = os.path.join(conan_file_path, conanfile_name) - if ((os.path.exists(conan_file_path) and conanfile_name not in os.listdir(src_folder)) or - (conanfile_name != "conanfile.py" and conanfile_name.lower() == "conanfile.py")): - raise ConanException("Wrong '%s' case" % conanfile_name) - conan_linter(conan_file_path, self._user_io.out) - conanfile = load_export_conanfile(conan_file_path, self._user_io.out, name, version) - conan_ref = ConanFileReference(conanfile.name, conanfile.version, user, channel) - conan_ref_str = str(conan_ref) - # Maybe a platform check could be added, but depends on disk partition - refs = self._search_manager.search(conan_ref_str, ignorecase=True) - if refs and conan_ref not in refs: - raise ConanException("Cannot export package with same name but different case\n" - "You exported '%s' but already existing '%s'" - % (conan_ref_str, " ".join(str(s) for s in refs))) - output = ScopedOutput(str(conan_ref), self._user_io.out) - with self._client_cache.conanfile_write_lock(conan_ref): - export_conanfile(output, self._client_cache, conanfile, src_folder, conan_ref, keep_source, - filename) + from conans.client.cmd.export import cmd_export + cmd_export(user, channel, conan_file_path, self._user_io.out, self._search_manager, + self._client_cache, keep_source, filename, name, version) def export_pkg(self, reference, source_folder, build_folder, install_folder, profile, force): @@ -585,30 +555,6 @@ def search_packages(self, reference=None, remote=None, packages_query=None, outd return ordered_packages, reference, recipe_hash, packages_query - def copy(self, reference, package_ids, username, channel, force=False): - """ Copy or move conanfile (exported) and packages to another user and or channel - @param reference: ConanFileReference containing the packages to be moved - @param package_ids: list of ids or [] for all list - @param username: Destination username - @param channel: Destination channel - @param remote: install only from that remote - """ - # It is necessary to make sure the sources are complete before proceeding - remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, None) - - # Now we can actually copy - conan_file_path = self._client_cache.conanfile(reference) - conanfile = load_conanfile_class(conan_file_path) - remote_proxy.complete_recipe_sources(reference, short_paths=conanfile.short_paths) - copier = PackageCopier(self._client_cache, self._user_io, conanfile.short_paths) - if not package_ids: - packages = self._client_cache.packages(reference) - if os.path.exists(packages): - package_ids = os.listdir(packages) - else: - package_ids = [] - copier.copy(reference, package_ids, username, channel, force) - def remove(self, pattern, src=False, build_ids=None, package_ids_filter=None, force=False, remote=None, packages_query=None, outdated=False): """ Remove conans and/or packages diff --git a/conans/client/package_copier.py b/conans/client/package_copier.py deleted file mode 100644 index 036be74196c..00000000000 --- a/conans/client/package_copier.py +++ /dev/null @@ -1,52 +0,0 @@ -from conans.model.ref import ConanFileReference, PackageReference -import os -from conans.util.files import rmdir -import shutil -from conans.errors import ConanException - - -class PackageCopier(object): - """ Class responsible for copying or moving packages from users/channels """ - - def __init__(self, paths, user_io, short_paths=False): - self._user_io = user_io - self._paths = paths - self._short_paths = short_paths - - def copy(self, reference, package_ids, username, channel, force=False): - assert(isinstance(reference, ConanFileReference)) - dest_ref = ConanFileReference(reference.name, reference.version, username, channel) - # Copy export - export_origin = self._paths.export(reference) - if not os.path.exists(export_origin): - raise ConanException("'%s' doesn't exist" % str(reference)) - export_dest = self._paths.export(dest_ref) - if os.path.exists(export_dest): - if not force and not self._user_io.request_boolean("'%s' already exist. Override?" - % str(dest_ref)): - return - rmdir(export_dest) - shutil.copytree(export_origin, export_dest, symlinks=True) - self._user_io.out.info("Copied %s to %s" % (str(reference), str(dest_ref))) - - export_sources_origin = self._paths.export_sources(reference, self._short_paths) - export_sources_dest = self._paths.export_sources(dest_ref, self._short_paths) - if os.path.exists(export_sources_dest): - rmdir(export_sources_dest) - shutil.copytree(export_sources_origin, export_sources_dest, symlinks=True) - self._user_io.out.info("Copied sources %s to %s" % (str(reference), str(dest_ref))) - - # Copy packages - for package_id in package_ids: - package_origin = PackageReference(reference, package_id) - package_dest = PackageReference(dest_ref, package_id) - package_path_origin = self._paths.package(package_origin, self._short_paths) - package_path_dest = self._paths.package(package_dest, self._short_paths) - if os.path.exists(package_path_dest): - if not force and not self._user_io.request_boolean("Package '%s' already exist." - " Override?" - % str(package_id)): - continue - rmdir(package_path_dest) - shutil.copytree(package_path_origin, package_path_dest, symlinks=True) - self._user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref))) diff --git a/conans/test/functional/package_copier_test.py b/conans/test/functional/package_copier_test.py index 2d606e88c09..2ea9bd2772c 100644 --- a/conans/test/functional/package_copier_test.py +++ b/conans/test/functional/package_copier_test.py @@ -7,7 +7,7 @@ from conans.model.ref import ConanFileReference, PackageReference from conans.paths import SimplePaths import os -from conans.client.package_copier import PackageCopier +from conans.client.cmd.copy import package_copy class MockedBooleanUserIO(UserIO): @@ -27,7 +27,6 @@ def test_copy(self): output = TestBufferConanOutput() userio = MockedBooleanUserIO(True, out=output) paths = SimplePaths(temp_folder()) - copier = PackageCopier(paths, userio) # Create some packages to copy reference = ConanFileReference.loads("Hello/0.1@lasote/testing") @@ -36,7 +35,8 @@ def test_copy(self): self._create_package(reference, "2222222", paths) # Copy all to destination - copier.copy(reference, ["0101001", "2222222"], "lasote", "stable", force=False) + package_copy(reference, "lasote/stable", ["0101001", "2222222"], paths, + user_io=userio, force=False) new_reference = ConanFileReference.loads("Hello/0.1@lasote/stable") self._assert_conanfile_exists(new_reference, paths) self._assert_package_exists(new_reference, "0101001", paths) @@ -47,7 +47,8 @@ def test_copy(self): # Copy again, without force and answering yes output._stream.truncate(0) # Reset output - copier.copy(reference, ["0101001", "2222222"], "lasote", "stable", force=False) + package_copy(reference, "lasote/stable", ["0101001", "2222222"], paths, + user_io=userio, force=False) self.assertIn("Copied Hello/0.1@lasote/testing to Hello/0.1@lasote/stable", output) self.assertIn("Copied 0101001 to Hello/0.1@lasote/stable", output) self.assertIn("Copied 2222222 to Hello/0.1@lasote/stable", output) @@ -60,7 +61,8 @@ def test_copy(self): self._create_package(reference, "0101001", paths, "new lib content") self._create_package(reference, "2222222", paths, "new lib content") output._stream.truncate(0) # Reset output - copier.copy(reference, ["0101001", "2222222"], "lasote", "stable", force=False) + package_copy(reference, "lasote/stable", ["0101001", "2222222"], paths, + user_io=userio, force=False) conanfile_content = load(os.path.join(paths.export(new_reference), "conanfile.py")) self.assertEquals(conanfile_content, "new content") package_content = load(os.path.join(paths.package(PackageReference(new_reference, "0101001")), @@ -70,12 +72,12 @@ def test_copy(self): # Now we are going to answer always NO to override output._stream.truncate(0) # Reset output userio = MockedBooleanUserIO(False, out=output) - copier = PackageCopier(paths, userio) self._create_conanfile(reference, paths, "content22") self._create_package(reference, "0101001", paths, "newlib22") self._create_package(reference, "2222222", paths, "newlib22") - copier.copy(reference, ["0101001", "2222222"], "lasote", "stable", force=False) + package_copy(reference, "lasote/stable", ["0101001", "2222222"], paths, + user_io=userio, force=False) conanfile_content = load(os.path.join(paths.export(new_reference), "conanfile.py")) self.assertEquals(conanfile_content, "new content") # Not content22 p_ref = PackageReference(new_reference, "0101001") @@ -89,13 +91,15 @@ def test_copy(self): # Now override output._stream.truncate(0) # Reset output - copier.copy(reference, ["0101001", "2222222"], "lasote", "stable", force=True) + package_copy(reference, "lasote/stable", ["0101001", "2222222"], paths, + user_io=userio, force=True) self.assertIn("Copied 0101001 to Hello/0.1@lasote/stable", output) self.assertIn("Copied 2222222 to Hello/0.1@lasote/stable", output) # Now copy just one package to another user/channel output._stream.truncate(0) # Reset output - copier.copy(reference, ["0101001"], "pepe", "mychannel", force=True) + package_copy(reference, "pepe/mychannel", ["0101001"], paths, + user_io=userio, force=True) self.assertIn("Copied 0101001 to Hello/0.1@pepe/mychannel", output) self.assertNotIn("Copied 2222222 to Hello/0.1@pepe/mychannel", output) new_reference = ConanFileReference.loads("Hello/0.1@pepe/mychannel") diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py index 707eda9d3a7..c01f2fbfd08 100644 --- a/conans/test/utils/tools.py +++ b/conans/test/utils/tools.py @@ -13,7 +13,7 @@ from six.moves.urllib.parse import urlsplit, urlunsplit from webtest.app import TestApp -from conans import __version__ as CLIENT_VERSION, tools +from conans import __version__ as CLIENT_VERSION from conans.client import settings_preprocessor from conans.client.client_cache import ClientCache from conans.client.command import Command From 995b629739d1d8ff1fe1f4ccbfba5492261903e4 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 10:04:11 +0100 Subject: [PATCH 11/38] added messages to concurrent locks (#2012) --- conans/client/client_cache.py | 6 ++++-- conans/util/locks.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/conans/client/client_cache.py b/conans/client/client_cache.py index 332416cb480..c56e51a61c6 100644 --- a/conans/client/client_cache.py +++ b/conans/client/client_cache.py @@ -46,12 +46,14 @@ def _no_locks(self): def conanfile_read_lock(self, conan_ref): if self._no_locks(): return NoLock() - return ReadLock(os.path.join(self.conan(conan_ref), "rw")) + return ReadLock(os.path.join(self.conan(conan_ref), "rw"), conan_ref, + self._output) def conanfile_write_lock(self, conan_ref): if self._no_locks(): return NoLock() - return WriteLock(os.path.join(self.conan(conan_ref), "rw")) + return WriteLock(os.path.join(self.conan(conan_ref), "rw"), conan_ref, + self._output) def package_lock(self, package_ref): if self._no_locks(): diff --git a/conans/util/locks.py b/conans/util/locks.py index e1505550b11..59a465655a8 100644 --- a/conans/util/locks.py +++ b/conans/util/locks.py @@ -31,9 +31,20 @@ def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable class Lock(object): - def __init__(self, folder): + def __init__(self, folder, locked_item, output): self._count_file = folder + ".count" self._count_lock_file = folder + ".count.lock" + self._locked_item = locked_item + self._output = output + self._first_lock = True + + def _info_locked(self): + if self._first_lock: + self._first_lock = False + self._output.info("%s is locked by another concurrent conan process, wait..." + % str(self._locked_item)) + self._output.info("If not the case, quit, and do 'conan remove %s -f'" + % str(self._locked_item)) def _readers(self): try: @@ -51,6 +62,7 @@ def __enter__(self): if readers >= 0: save(self._count_file, str(readers + 1)) break + self._info_locked() time.sleep(READ_BUSY_DELAY) def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable @@ -68,6 +80,7 @@ def __enter__(self): if readers == 0: save(self._count_file, "-1") break + self._info_locked() time.sleep(WRITE_BUSY_DELAY) def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable From c677dd3532f19268f51dbe8013df12eb0f22a83e Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 10:07:07 +0100 Subject: [PATCH 12/38] fixing appending while download overwrite (#2013) --- conans/client/rest/uploader_downloader.py | 2 +- conans/test/util/tools_test.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conans/client/rest/uploader_downloader.py b/conans/client/rest/uploader_downloader.py index 8caeb06bb66..7b0df54fc6f 100644 --- a/conans/client/rest/uploader_downloader.py +++ b/conans/client/rest/uploader_downloader.py @@ -174,7 +174,7 @@ def download_chunks(file_handler=None, ret_buffer=None): if file_path: mkdir(os.path.dirname(file_path)) - with open(file_path, 'ab') as handle: + with open(file_path, 'wb') as handle: dl_size = download_chunks(file_handler=handle) else: dl_size = download_chunks(ret_buffer=ret) diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py index 0f3b59ed01c..7b84a965d2b 100644 --- a/conans/test/util/tools_test.py +++ b/conans/test/util/tools_test.py @@ -16,12 +16,12 @@ from conans.paths import CONANFILE from conans.test.utils.runner import TestRunner from conans.test.utils.test_files import temp_folder -from conans.test.utils.tools import TestClient, TestBufferConanOutput, TestRequester +from conans.test.utils.tools import TestClient, TestBufferConanOutput from conans.test.utils.visual_project_files import get_vs_project_files from conans.test.utils.context_manager import which from conans.tools import OSInfo, SystemPackageTool, replace_in_file, AptTool, ChocolateyTool,\ set_global_instances -from conans.util.files import save +from conans.util.files import save, load import requests @@ -481,6 +481,7 @@ def download_retries_test(self): retry=3, retry_wait=0) self.assertTrue(os.path.exists(dest)) + content = load(dest) # overwrite = False with self.assertRaises(ConanException): @@ -494,6 +495,8 @@ def download_retries_test(self): retry=3, retry_wait=0, overwrite=True) self.assertTrue(os.path.exists(dest)) + content_new = load(dest) + self.assertEqual(content, content_new) # Not authorized with self.assertRaises(ConanException): From 7be7840ae6e550af686354635ee55fdd56699c3d Mon Sep 17 00:00:00 2001 From: SSE4 Date: Mon, 13 Nov 2017 16:12:47 +0700 Subject: [PATCH 13/38] Autotools fix (#2014) * - use "--host=" instead of "--host " syntax * - use correct host and build for Apple (config.guess returns apple-darwin) * - additional tests for AutoToolsBuildEnvironment and host/build/target --- conans/client/build/autotools_environment.py | 16 +++++---- .../functional/autotools_configure_test.py | 36 +++++++++++++++++-- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/conans/client/build/autotools_environment.py b/conans/client/build/autotools_environment.py index cf877891546..0dc8ef0a6e5 100644 --- a/conans/client/build/autotools_environment.py +++ b/conans/client/build/autotools_environment.py @@ -88,7 +88,7 @@ def _get_host_build_target_flags(self, arch_detected, os_detected): host = "i686-w64-mingw32" if arch_setting == "x86" else "x86_64-w64-mingw32" else: # Building for Linux or Android build = "%s-%s" % (arch_detected, {"Linux": "linux-gnu", - "Darwin": "apple-macos"}.get(os_detected, + "Darwin": "apple-darwin"}.get(os_detected, os_detected.lower())) if arch_setting == "x86": host_arch = "i686" @@ -97,8 +97,12 @@ def _get_host_build_target_flags(self, arch_detected, os_detected): else: host_arch = "arm" if "arm" in arch_setting else arch_setting - host = "%s%s" % (host_arch, { "Linux": "-linux-gnueabi", - "Android": "-linux-android"}.get(os_setting, "")) + host = "%s%s" % (host_arch, {"Linux": "-linux-gnueabi", + "Android": "-linux-android", + "Macos": "-apple-darwin", + "iOS": "-apple-darwin", + "watchOS": "-apple-darwin", + "tvOS": "-apple-darwin"}.get(os_setting, "")) if arch_setting == "armv7hf" and os_setting == "Linux": host += "hf" elif "arm" in arch_setting and arch_setting != "armv8" and os_setting == "Android": @@ -133,15 +137,15 @@ def configure(self, configure_dir=None, args=None, build=None, host=None, target if build is not False: # Skipped by user if build or auto_build: # User specified value or automatic - triplet_args.append("--build %s" % (build or auto_build)) + triplet_args.append("--build=%s" % (build or auto_build)) if host is not False: # Skipped by user if host or auto_host: # User specified value or automatic - triplet_args.append("--host %s" % (host or auto_host)) + triplet_args.append("--host=%s" % (host or auto_host)) if target is not False: # Skipped by user if target or auto_target: # User specified value or automatic - triplet_args.append("--target %s" % (target or auto_target)) + triplet_args.append("--target=%s" % (target or auto_target)) with environment_append(self.vars): self._conanfile.run("%s/configure %s %s" diff --git a/conans/test/functional/autotools_configure_test.py b/conans/test/functional/autotools_configure_test.py index 782d9ac4fc9..7ca1b9df6b0 100644 --- a/conans/test/functional/autotools_configure_test.py +++ b/conans/test/functional/autotools_configure_test.py @@ -349,5 +349,37 @@ def get_values(this_os, this_arch, setting_os, setting_arch): self.assertFalse(target) build, host, target = get_values("Darwin", "x86_64", "Android", "armv7hf") - self.assertEquals(build, "x86_64-apple-macos") - self.assertEquals(host, "arm-linux-androideabi") \ No newline at end of file + self.assertEquals(build, "x86_64-apple-darwin") + self.assertEquals(host, "arm-linux-androideabi") + + build, host, target = get_values("Darwin", "x86_64", "Macos", "x86") + self.assertEquals(build, "x86_64-apple-darwin") + self.assertEquals(host, "i686-apple-darwin") + + build, host, target = get_values("Darwin", "x86_64", "iOS", "armv7") + self.assertEquals(build, "x86_64-apple-darwin") + self.assertEquals(host, "arm-apple-darwin") + + build, host, target = get_values("Darwin", "x86_64", "watchOS", "armv7k") + self.assertEquals(build, "x86_64-apple-darwin") + self.assertEquals(host, "arm-apple-darwin") + + build, host, target = get_values("Darwin", "x86_64", "tvOS", "arm64") + self.assertEquals(build, "x86_64-apple-darwin") + self.assertEquals(host, "arm-apple-darwin") + + def cross_build_command_test(self): + runner = RunnerMock() + conanfile = MockConanfile(MockSettings({}), runner) + ab = AutoToolsBuildEnvironment(conanfile) + ab.configure() + self.assertEquals(runner.command_called, "./configure ") + + ab.configure(host="x86_64-apple-darwin") + self.assertEquals(runner.command_called, "./configure --host=x86_64-apple-darwin") + + ab.configure(build="arm-apple-darwin") + self.assertEquals(runner.command_called, "./configure --build=arm-apple-darwin") + + ab.configure(target="i686-apple-darwin") + self.assertEquals(runner.command_called, "./configure --target=i686-apple-darwin") From 81d1fcbf8fc8530160b2210e25c2720b316b6f36 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Mon, 13 Nov 2017 16:40:15 +0700 Subject: [PATCH 14/38] - CMake parallel build for Visual Studio (#1965) --- conans/client/build/cmake.py | 4 +++ conans/test/functional/cmake_test.py | 39 ++++++++++++++++++---------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/conans/client/build/cmake.py b/conans/client/build/cmake.py index be1cd84fb5c..2e5559d2402 100644 --- a/conans/client/build/cmake.py +++ b/conans/client/build/cmake.py @@ -364,6 +364,10 @@ def _build_new(self, args=None, build_dir=None, target=None): if "--" not in args: args.append("--") args.append("-j%i" % cpu_count()) + elif "Visual Studio" in self.generator: + if "--" not in args: + args.append("--") + args.append("/m:%i" % cpu_count()) arg_list = join_arguments([ args_to_string([build_dir]), diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 50a81b89c67..d8b884df214 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -245,9 +245,12 @@ def test_deprecated_behaviour(self): conan_file.command) cmake.build(conan_file) - self.assertEqual('cmake --build %s' % dot_dir, conan_file.command) + self.assertEqual('cmake --build %s %s' + % (dot_dir, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.test() - self.assertEqual('cmake --build %s %s' % (dot_dir, CMakeTest.scape('--target RUN_TESTS')), + self.assertEqual('cmake --build %s %s %s' + % (dot_dir, CMakeTest.scape('--target RUN_TESTS'), + (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) def convenient_functions_test(self): @@ -282,18 +285,22 @@ def convenient_functions_test(self): conan_file.command) cmake.build() - self.assertEqual('cmake --build %s' % dot_dir, conan_file.command) + self.assertEqual('cmake --build %s %s' % + (dot_dir, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.test() - self.assertEqual('cmake --build %s %s' % (dot_dir, target_test), conan_file.command) + self.assertEqual('cmake --build %s %s %s' % + (dot_dir, target_test, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) settings.build_type = "Debug" cmake = CMake(conan_file) cmake.build() - self.assertEqual('cmake --build %s --config Debug' % dot_dir, conan_file.command) + self.assertEqual('cmake --build %s --config Debug %s' % + (dot_dir,(CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.test() - self.assertEqual('cmake --build %s --config Debug %s' % (dot_dir, target_test), conan_file.command) + self.assertEqual('cmake --build %s --config Debug %s %s' % + (dot_dir, target_test, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.configure(source_dir="/source", build_dir=self.tempdir, args=['--foo "bar"'], defs={"SHARED": True}) @@ -312,30 +319,34 @@ def convenient_functions_test(self): escaped_args = '--target install "--bar \'foo\'"' else: escaped_args = r"'--target' 'install' '--bar '\''foo'\'''" - self.assertEqual('cmake --build %s --config Debug %s' % (tempdir, escaped_args), - conan_file.command) + self.assertEqual('cmake --build %s --config Debug %s %s' + % (tempdir, escaped_args, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.test(args=["--bar 'foo'"]) if sys.platform == 'win32': escaped_args = '%s "--bar \'foo\'"' % target_test else: escaped_args = r"%s '--bar '\''foo'\'''" % target_test - self.assertEqual('cmake --build %s --config Debug %s' % (tempdir, escaped_args), - conan_file.command) + self.assertEqual('cmake --build %s --config Debug %s %s' % + (tempdir, escaped_args, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) settings.build_type = "Release" cmake = CMake(conan_file) cmake.build() - self.assertEqual('cmake --build %s --config Release' % dot_dir, conan_file.command) + self.assertEqual('cmake --build %s --config Release %s' % + (dot_dir, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.test() - self.assertEqual('cmake --build %s --config Release %s' % (dot_dir, target_test), conan_file.command) + self.assertEqual('cmake --build %s --config Release %s %s' + % (dot_dir, target_test, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.build(build_dir=self.tempdir) - self.assertEqual('cmake --build %s --config Release' % tempdir, conan_file.command) + self.assertEqual('cmake --build %s --config Release %s' + % (tempdir, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) cmake.test(build_dir=self.tempdir) - self.assertEqual('cmake --build %s --config Release %s' % (tempdir, target_test), conan_file.command) + self.assertEqual('cmake --build %s --config Release %s %s' + % (tempdir, target_test, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) settings.compiler = "gcc" settings.compiler.version = "5.4" From 2ac536f096dc30f01960def4bcc924b05be1fa33 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Mon, 13 Nov 2017 12:23:23 +0100 Subject: [PATCH 15/38] add boost build generator (#1800) * add boost build generator * Replace slases and little test * Trailing endline --- conans/client/generators/__init__.py | 2 + conans/client/generators/boostbuild.py | 51 +++++++++++++++++ conans/test/generators/boost_build_test.py | 66 ++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 conans/client/generators/boostbuild.py create mode 100644 conans/test/generators/boost_build_test.py diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index 5ba5a5e0c7c..bd02dea4481 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -19,6 +19,7 @@ from .virtualenv import VirtualEnvGenerator from .cmake_multi import CMakeMultiGenerator from .virtualbuildenv import VirtualBuildEnvGenerator +from .boostbuild import BoostBuildGenerator class _GeneratorManager(object): @@ -57,6 +58,7 @@ def __getitem__(self, key): registered_generators.add("virtualenv", VirtualEnvGenerator) registered_generators.add("virtualbuildenv", VirtualBuildEnvGenerator) registered_generators.add("virtualrunenv", VirtualRunEnvGenerator) +registered_generators.add("boost-build", BoostBuildGenerator) registered_generators.add("pkg_config", PkgConfigGenerator) diff --git a/conans/client/generators/boostbuild.py b/conans/client/generators/boostbuild.py new file mode 100644 index 00000000000..f7e98a77fc0 --- /dev/null +++ b/conans/client/generators/boostbuild.py @@ -0,0 +1,51 @@ + +""" +Boost Build Conan Generator + +This is a simple project-root.jam generator declaring all conan dependencies +as boost-build lib targets. This lets you link against them in your Jamfile +as a property. Link against the "conant-deps" target. + +""" + +from conans.model import Generator + + +def JamfileOutput(dep_cpp_info): + out = '' + for lib in dep_cpp_info.libs: + out += 'lib %s :\n' % lib + out += '\t: # requirements\n' + out += '\t%s\n' % lib + out += ''.join('\t%s\n' % x.replace("\\", "/") for x in dep_cpp_info.lib_paths) + out += '\t: # default-build\n' + out += '\t: # usage-requirements\n' + out += ''.join('\t%s\n' % x for x in dep_cpp_info.defines) + out += ''.join('\t%s\n' % x.replace("\\", "/") for x in dep_cpp_info.include_paths) + out += ''.join('\t%s\n' % x for x in dep_cpp_info.cppflags) + out += ''.join('\t%s\n' % x for x in dep_cpp_info.cflags) + out += ''.join('\t%s\n' % x for x in dep_cpp_info.sharedlinkflags) + out += '\t;\n\n' + return out + + +class BoostBuildGenerator(Generator): + @property + def filename(self): + return "project-root.jam" + + @property + def content(self): + out = '' + + for dep_name, dep_cpp_info in self.deps_build_info.dependencies: + out += JamfileOutput(dep_cpp_info) + + out += 'alias conan-deps :\n' + for dep_name, dep_cpp_info in self.deps_build_info.dependencies: + for lib in dep_cpp_info.libs: + out += '\t%s\n' % lib + out += ';\n' + + return out + diff --git a/conans/test/generators/boost_build_test.py b/conans/test/generators/boost_build_test.py new file mode 100644 index 00000000000..c34682a32ba --- /dev/null +++ b/conans/test/generators/boost_build_test.py @@ -0,0 +1,66 @@ +import unittest + +from conans.client.generators.boostbuild import BoostBuildGenerator +from conans.model.settings import Settings +from conans.model.conan_file import ConanFile +from conans.model.build_info import CppInfo +from conans.model.ref import ConanFileReference + + +class BoostJamGeneratorTest(unittest.TestCase): + + def variables_setup_test(self): + + conanfile = ConanFile(None, None, Settings({}), None) + + ref = ConanFileReference.loads("MyPkg/0.1@lasote/stables") + cpp_info = CppInfo("dummy_root_folder1") + cpp_info.defines = ["MYDEFINE1"] + cpp_info.cflags.append("-Flag1=23") + cpp_info.version = "1.3" + cpp_info.description = "My cool description" + cpp_info.libs = ["MyLib1"] + + conanfile.deps_cpp_info.update(cpp_info, ref.name) + ref = ConanFileReference.loads("MyPkg2/0.1@lasote/stables") + cpp_info = CppInfo("dummy_root_folder2") + cpp_info.libs = ["MyLib2"] + cpp_info.defines = ["MYDEFINE2"] + cpp_info.version = "2.3" + cpp_info.exelinkflags = ["-exelinkflag"] + cpp_info.sharedlinkflags = ["-sharedlinkflag"] + cpp_info.cppflags = ["-cppflag"] + cpp_info.public_deps = ["MyPkg"] + cpp_info.lib_paths.extend(["Path\\with\\slashes", "regular/path/to/dir"]) + cpp_info.include_paths.extend(["other\\Path\\with\\slashes", "other/regular/path/to/dir"]) + conanfile.deps_cpp_info.update(cpp_info, ref.name) + generator = BoostBuildGenerator(conanfile) + + self.assertEquals(generator.content, """lib MyLib1 : + : # requirements + MyLib1 + : # default-build + : # usage-requirements + MYDEFINE1 + -Flag1=23 + ; + +lib MyLib2 : + : # requirements + MyLib2 + Path/with/slashes + regular/path/to/dir + : # default-build + : # usage-requirements + MYDEFINE2 + other/Path/with/slashes + other/regular/path/to/dir + -cppflag + -sharedlinkflag + ; + +alias conan-deps : + MyLib1 + MyLib2 +; +""") From 937ffac349acdfecfe88bbbe228cedb840bd22a9 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Date: Mon, 13 Nov 2017 14:35:49 +0100 Subject: [PATCH 16/38] Moved import to top file --- conans/client/manager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conans/client/manager.py b/conans/client/manager.py index 77fd7e815ee..03e269605aa 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -4,9 +4,11 @@ from collections import OrderedDict, Counter from conans.client import packager +from conans.client.build_requires import BuildRequires from conans.client.client_cache import ClientCache -from conans.client.deps_builder import DepsGraphBuilder +from conans.client.cmd.export import cmd_export from conans.client.conf.detect import detected_os +from conans.client.deps_builder import DepsGraphBuilder from conans.client.generators import write_generators from conans.client.generators.text import TXTGenerator from conans.client.importer import run_imports, undo_imports, run_deploy @@ -24,14 +26,13 @@ from conans.client.uploader import ConanUploader from conans.client.userio import UserIO from conans.errors import NotFoundException, ConanException, conanfile_exception_formatter +from conans.model.manifest import FileTreeManifest from conans.model.ref import ConanFileReference, PackageReference from conans.paths import CONANFILE, CONANINFO, CONANFILE_TXT, CONAN_MANIFEST, BUILD_INFO +from conans.search.search import filter_outdated from conans.tools import environment_append -from conans.util.files import save, rmdir, normalize, mkdir, load +from conans.util.files import save, rmdir, normalize, mkdir, load from conans.util.log import logger -from conans.model.manifest import FileTreeManifest -from conans.client.build_requires import BuildRequires -from conans.search.search import filter_outdated class BuildMode(object): @@ -134,7 +135,6 @@ def get_loader(self, profile): def export(self, user, channel, conan_file_path, keep_source=False, filename=None, name=None, version=None): - from conans.client.cmd.export import cmd_export cmd_export(user, channel, conan_file_path, self._user_io.out, self._search_manager, self._client_cache, keep_source, filename, name, version) From 291cccd2141b7da86508885b3ceb79dcffe0d844 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Mon, 13 Nov 2017 14:54:09 +0100 Subject: [PATCH 17/38] install-folder in deploy and controlled folders presence in package_info (#2018) --- conans/client/conan_api.py | 2 + conans/client/importer.py | 12 +++--- conans/client/installer.py | 3 ++ conans/test/functional/deploy_test.py | 41 +++++++++++-------- conans/test/functional/folders_access_test.py | 21 ++++++---- 5 files changed, 50 insertions(+), 29 deletions(-) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index ebea6242b67..09c365e98e6 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -456,6 +456,8 @@ def install_reference(self, reference, settings=None, options=None, env=None, sc if not generators: # We don't want the default txt generators = False + + mkdir(install_folder) self._manager.install(reference=reference, install_folder=install_folder, remote=remote, profile=profile, build_modes=build, update=update, manifest_folder=manifest_folder, diff --git a/conans/client/importer.py b/conans/client/importer.py index 24bbf063a9b..a77569c1f18 100644 --- a/conans/client/importer.py +++ b/conans/client/importer.py @@ -74,26 +74,28 @@ def run_imports(conanfile, dest_folder, output): return copied_files -def run_deploy(conanfile, dest_folder, output): +def run_deploy(conanfile, install_folder, output): deploy_output = ScopedOutput("%s deploy()" % output.scope, output) - file_importer = _FileImporter(conanfile, dest_folder) + file_importer = _FileImporter(conanfile, install_folder) package_copied = set() # This is necessary to capture FileCopier full destination paths # Maybe could be improved in FileCopier def file_copier(*args, **kwargs): - file_copy = FileCopier(conanfile.package_folder, dest_folder) + file_copy = FileCopier(conanfile.package_folder, install_folder) copied = file_copy(*args, **kwargs) package_copied.update(copied) conanfile.copy_deps = file_importer conanfile.copy = file_copier + conanfile.install_folder = install_folder with environment_append(conanfile.env): - conanfile.deploy() + with tools.chdir(install_folder): + conanfile.deploy() copied_files = file_importer.copied_files copied_files.update(package_copied) - _report_save_manifest(copied_files, deploy_output, dest_folder, "deploy_manifest.txt") + _report_save_manifest(copied_files, deploy_output, install_folder, "deploy_manifest.txt") class _FileImporter(object): diff --git a/conans/client/installer.py b/conans/client/installer.py index f7829c5021d..909492d4d76 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -260,6 +260,9 @@ def call_package_info(conanfile, package_folder): # package folder and artifacts with tools.chdir(package_folder): with conanfile_exception_formatter(str(conanfile), "package_info"): + conanfile.source_folder = None + conanfile.build_folder = None + conanfile.install_folder = None conanfile.package_info() diff --git a/conans/test/functional/deploy_test.py b/conans/test/functional/deploy_test.py index 731d1a78b20..f608f79edac 100644 --- a/conans/test/functional/deploy_test.py +++ b/conans/test/functional/deploy_test.py @@ -50,21 +50,28 @@ def deploy(self): client.save({"conanfile.py": conanfile}) client.run("create Pkg/0.1@user/testing") self.assertNotIn("deploy()", client.out) - client.current_folder = temp_folder() - client.run("install Pkg/0.1@user/testing") - self.assertIn("Pkg/0.1@user/testing deploy(): Copied 1 '.dll' files: mylib.dll", - client.out) - self.assertIn("Pkg/0.1@user/testing deploy(): Copied 1 '.exe' files: myapp.exe", - client.out) - deploy_manifest = FileTreeManifest.loads(load(os.path.join(client.current_folder, - "deploy_manifest.txt"))) - app = os.path.join(client.current_folder, "myapp.exe") - if deploy_to_abs: - lib = os.path.join(dll_folder, "mylib.dll") - else: - lib = os.path.join(client.current_folder, "mylib.dll") - self.assertEqual(sorted([app, lib]), - sorted(deploy_manifest.file_sums.keys())) - self.assertEqual(load(app), "myexe") - self.assertEqual(load(lib), "mydll") + def test_install_in(folder): + client.current_folder = temp_folder() + client.run("install Pkg/0.1@user/testing --install-folder=%s" % folder) + + self.assertIn("Pkg/0.1@user/testing deploy(): Copied 1 '.dll' files: mylib.dll", + client.out) + self.assertIn("Pkg/0.1@user/testing deploy(): Copied 1 '.exe' files: myapp.exe", + client.out) + deploy_manifest = FileTreeManifest.loads(load(os.path.join(client.current_folder, + folder, + "deploy_manifest.txt"))) + + app = os.path.abspath(os.path.join(client.current_folder, folder, "myapp.exe")) + if deploy_to_abs: + lib = os.path.join(dll_folder, "mylib.dll") + else: + lib = os.path.abspath(os.path.join(client.current_folder, folder, "mylib.dll")) + self.assertEqual(sorted([app, lib]), + sorted(deploy_manifest.file_sums.keys())) + self.assertEqual(load(app), "myexe") + self.assertEqual(load(lib), "mydll") + + test_install_in("./") + test_install_in("other_install_folder") diff --git a/conans/test/functional/folders_access_test.py b/conans/test/functional/folders_access_test.py index 5a817e7b22e..1c1aecf293e 100644 --- a/conans/test/functional/folders_access_test.py +++ b/conans/test/functional/folders_access_test.py @@ -100,17 +100,17 @@ def package_info(self): assert(self.package_folder == os.getcwd()) assert(self.in_local_cache == True) - if self.no_copy_source: - assert(self.copy_source_folder == self.source_folder) - else: - assert(self.source_folder == self.build_folder) - - assert(self.copy_package_folder == self.package_folder) - assert(self.copy_build_folder == self.build_folder) + assert(self.source_folder is None) + assert(self.build_folder is None) + assert(self.install_folder is None) + def imports(self): assert(self.imports_folder == os.getcwd()) + def deploy(self): + assert(self.install_folder == os.getcwd()) + """ @@ -204,6 +204,13 @@ def imports_local_test(self): self.assertTrue(error) self.assertIn("ERROR: conanbuildinfo.txt file not found", self.client.out) + def deploy_test(self): + c1 = conanfile % {"no_copy_source": False, "source_with_infos": True, + "local_command": False} + self.client.save({"conanfile.py": c1}, clean_first=True) + self.client.run("create user/testing --build missing") + self.client.run("install lib/1.0@user/testing") # Checks deploy + def full_install_test(self): c1 = conanfile % {"no_copy_source": False, "source_with_infos": False, "local_command": False} From f85e740c616a27eb8be3ff2d38b38a1aaf4d0182 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Mon, 13 Nov 2017 22:23:07 +0700 Subject: [PATCH 18/38] - fix CMake.test() for Ninja generator (#2015) --- conans/client/build/cmake.py | 2 +- conans/test/functional/cmake_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/conans/client/build/cmake.py b/conans/client/build/cmake.py index 2e5559d2402..2720209b704 100644 --- a/conans/client/build/cmake.py +++ b/conans/client/build/cmake.py @@ -388,7 +388,7 @@ def test(self, args=None, build_dir=None, target=None): if isinstance(args, ConanFile): raise ConanException(deprecated_conanfile_param_message) if not target: - target = "RUN_TESTS" if self._compiler == "Visual Studio" else "test" + target = "RUN_TESTS" if self.is_multi_configuration else "test" self._build_new(args=args, build_dir=build_dir, target=target) @property diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index d8b884df214..079c1fdff46 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -370,6 +370,31 @@ def convenient_functions_test(self): cmake.test() self.assertEqual('cmake --build %s' % CMakeTest.scape('. --target test'), conan_file.command) + def test_run_tests(self): + settings = Settings.loads(default_settings_yml) + settings.os = "Windows" + settings.compiler = "Visual Studio" + settings.compiler.version = "12" + settings.compiler.runtime = "MDd" + settings.arch = "x86" + settings.build_type = None + + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + cmake.test() + self.assertEqual('cmake --build %s' % CMakeTest.scape('. --target RUN_TESTS'), conan_file.command) + + cmake.generator = "Ninja Makefiles" + cmake.test() + self.assertEqual('cmake --build %s' % CMakeTest.scape('. --target test -- -j%i' % cpu_count()), + conan_file.command) + + cmake.generator = "NMake Makefiles" + cmake.test() + self.assertEqual('cmake --build %s' % CMakeTest.scape('. --target test -- -j%i' % cpu_count()), + conan_file.command) + def test_clean_sh_path(self): if platform.system() != "Windows": From dbcd28210c05eb819e59c6204b06c397f3cca86b Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 17:21:34 +0100 Subject: [PATCH 19/38] Feature/order cmake graph bug (#1973) * fixing bug in conanbuildinfo.cmake * working * passing tests * added back overall deps * merged develop * prefixed imported lib targets in cmake gen * fixed broken test --- conans/__init__.py | 1 - conans/client/generators/cmake_common.py | 50 +++++++++++++++----- conans/test/integration/private_deps_test.py | 8 ++-- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/conans/__init__.py b/conans/__init__.py index da04a9edbb2..01a6e25d2b2 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -17,4 +17,3 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] __version__ = '0.29.0-dev' - diff --git a/conans/client/generators/cmake_common.py b/conans/client/generators/cmake_common.py index 4e045a2e9d2..e4d1711c06b 100644 --- a/conans/client/generators/cmake_common.py +++ b/conans/client/generators/cmake_common.py @@ -105,22 +105,23 @@ def cmake_global_vars(deps, build_type=""): return _cmake_multi_dep_vars.format(cmd_line_args=cmd_line_args, deps=deps, build_type=_build_type_str(build_type)) + _target_template = """ - conan_find_libraries_abs_path("${{CONAN_LIBS_{uname}}}" "${{CONAN_LIB_DIRS_{uname}}}" - CONAN_FULLPATH_LIBS_{uname}) - conan_find_libraries_abs_path("${{CONAN_LIBS_{uname}_DEBUG}}" "${{CONAN_LIB_DIRS_{uname}_DEBUG}}" - CONAN_FULLPATH_LIBS_{uname}_DEBUG) - conan_find_libraries_abs_path("${{CONAN_LIBS_{uname}_RELEASE}}" "${{CONAN_LIB_DIRS_{uname}_RELEASE}}" - CONAN_FULLPATH_LIBS_{uname}_RELEASE) + conan_package_library_targets("${{CONAN_LIBS_{uname}}}" "${{CONAN_LIB_DIRS_{uname}}}" + CONAN_PACKAGE_TARGETS_{uname} "{deps}" "" {pkg_name}) + conan_package_library_targets("${{CONAN_LIBS_{uname}_DEBUG}}" "${{CONAN_LIB_DIRS_{uname}_DEBUG}}" + CONAN_PACKAGE_TARGETS_{uname}_DEBUG "{deps}" "debug" {pkg_name}) + conan_package_library_targets("${{CONAN_LIBS_{uname}_RELEASE}}" "${{CONAN_LIB_DIRS_{uname}_RELEASE}}" + CONAN_PACKAGE_TARGETS_{uname}_RELEASE "{deps}" "release" {pkg_name}) add_library({name} INTERFACE IMPORTED) # Property INTERFACE_LINK_FLAGS do not work, necessary to add to INTERFACE_LINK_LIBRARIES - set_property(TARGET {name} PROPERTY INTERFACE_LINK_LIBRARIES ${{CONAN_FULLPATH_LIBS_{uname}}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_LIST}} - $<$:${{CONAN_FULLPATH_LIBS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> - $<$:${{CONAN_FULLPATH_LIBS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> - $<$:${{CONAN_FULLPATH_LIBS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> - $<$:${{CONAN_FULLPATH_LIBS_{uname}_DEBUG}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_DEBUG_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_DEBUG_LIST}}> + set_property(TARGET {name} PROPERTY INTERFACE_LINK_LIBRARIES ${{CONAN_PACKAGE_TARGETS_{uname}}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_LIST}} + $<$:${{CONAN_PACKAGE_TARGETS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_PACKAGE_TARGETS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_PACKAGE_TARGETS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_PACKAGE_TARGETS_{uname}_DEBUG}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_DEBUG_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_DEBUG_LIST}}> {deps}) set_property(TARGET {name} PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${{CONAN_INCLUDE_DIRS_{uname}}} $<$:${{CONAN_INCLUDE_DIRS_{uname}_RELEASE}}> @@ -155,7 +156,7 @@ def generate_targets_section(dependencies): use_deps = ["CONAN_PKG::%s" % d for d in dep_info.public_deps] deps = "" if not use_deps else " ".join(use_deps) section.append(_target_template.format(name="CONAN_PKG::%s" % dep_name, deps=deps, - uname=dep_name.upper())) + uname=dep_name.upper(), pkg_name=dep_name)) all_targets = " ".join(["CONAN_PKG::%s" % name for name, _ in dependencies]) section.append(' set(CONAN_TARGETS %s)\n' % all_targets) @@ -168,7 +169,8 @@ def generate_targets_section(dependencies): function(conan_find_libraries_abs_path libraries package_libdir libraries_abs_path) foreach(_LIBRARY_NAME ${libraries}) unset(CONAN_FOUND_LIBRARY CACHE) - find_library(CONAN_FOUND_LIBRARY NAME ${_LIBRARY_NAME} PATHS ${package_libdir} NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) + find_library(CONAN_FOUND_LIBRARY NAME ${_LIBRARY_NAME} PATHS ${package_libdir} + NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) if(CONAN_FOUND_LIBRARY) message(STATUS "Library ${_LIBRARY_NAME} found ${CONAN_FOUND_LIBRARY}") set(CONAN_FULLPATH_LIBS ${CONAN_FULLPATH_LIBS} ${CONAN_FOUND_LIBRARY}) @@ -181,6 +183,28 @@ def generate_targets_section(dependencies): set(${libraries_abs_path} ${CONAN_FULLPATH_LIBS} PARENT_SCOPE) endfunction() +function(conan_package_library_targets libraries package_libdir libraries_abs_path deps build_type package_name) + foreach(_LIBRARY_NAME ${libraries}) + unset(CONAN_FOUND_LIBRARY CACHE) + find_library(CONAN_FOUND_LIBRARY NAME ${_LIBRARY_NAME} PATHS ${package_libdir} + NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) + if(CONAN_FOUND_LIBRARY) + message(STATUS "Library ${_LIBRARY_NAME} found ${CONAN_FOUND_LIBRARY}") + set(_LIB_NAME CONAN_LIB::${package_name}_${_LIBRARY_NAME}${build_type}) + add_library(${_LIB_NAME} UNKNOWN IMPORTED) + set_target_properties(${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${CONAN_FOUND_LIBRARY}) + string(REPLACE " " ";" deps_list "${deps}") + set_property(TARGET ${_LIB_NAME} PROPERTY INTERFACE_LINK_LIBRARIES ${deps_list}) + set(CONAN_FULLPATH_LIBS ${CONAN_FULLPATH_LIBS} ${_LIB_NAME}) + else() + message(STATUS "Library ${_LIBRARY_NAME} not found in package, might be system one") + set(CONAN_FULLPATH_LIBS ${CONAN_FULLPATH_LIBS} ${_LIBRARY_NAME}) + endif() + endforeach() + unset(CONAN_FOUND_LIBRARY CACHE) + set(${libraries_abs_path} ${CONAN_FULLPATH_LIBS} PARENT_SCOPE) +endfunction() + macro(conan_set_libcxx) if(DEFINED CONAN_LIBCXX) message(STATUS "Conan C++ stdlib: ${CONAN_LIBCXX}") diff --git a/conans/test/integration/private_deps_test.py b/conans/test/integration/private_deps_test.py index 7e3a454def3..33b3bf7a0e7 100644 --- a/conans/test/integration/private_deps_test.py +++ b/conans/test/integration/private_deps_test.py @@ -53,10 +53,10 @@ def modern_cmake_test(self): conanbuildinfo_cmake = load(os.path.join(self.client.current_folder, "conanbuildinfo.cmake")) conanbuildinfo_cmake = " ".join(conanbuildinfo_cmake.splitlines()) - self.assertRegexpMatches(conanbuildinfo_cmake, "CONAN_PKG::gf PROPERTY " - "INTERFACE_LINK_LIBRARIES .+CONAN_PKG::glew\)") - self.assertRegexpMatches(conanbuildinfo_cmake, "CONAN_PKG::ImGuiTest PROPERTY " - "INTERFACE_LINK_LIBRARIES .+CONAN_PKG::glm CONAN_PKG::gf\)") + self.assertIn("CONAN_PKG::gf PROPERTY INTERFACE_LINK_LIBRARIES " + "${CONAN_PACKAGE_TARGETS_GF}", conanbuildinfo_cmake) + self.assertIn("CONAN_PKG::ImGuiTest PROPERTY INTERFACE_LINK_LIBRARIES " + "${CONAN_PACKAGE_TARGETS_IMGUITEST}", conanbuildinfo_cmake) def consumer_force_build_test(self): """If a conanfile requires another private conanfile, but in the install is forced From 0503dbb6e3ee2b3bb65117aee0bd714033093682 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 17:27:36 +0100 Subject: [PATCH 20/38] fixed copy command, now able to not copy binaries (#2020) * fixed copy command, now able to not copy binaries * fixed broken tests --- conans/client/cmd/copy.py | 7 +++- conans/client/command.py | 5 +-- conans/test/command/copy_packages_test.py | 47 ++++++++++++----------- conans/test/functional/path_limit_test.py | 2 +- conans/test/integration/symlinks_test.py | 4 +- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/conans/client/cmd/copy.py b/conans/client/cmd/copy.py index 3f201e64dc2..bee0bf80f38 100644 --- a/conans/client/cmd/copy.py +++ b/conans/client/cmd/copy.py @@ -16,7 +16,9 @@ def _prepare_sources(client_cache, user_io, remote_manager, reference): def _get_package_ids(client_cache, reference, package_ids): - if not package_ids or package_ids is True: + if not package_ids: + return [] + if package_ids is True: packages = client_cache.packages(reference) if os.path.exists(packages): package_ids = os.listdir(packages) @@ -27,6 +29,9 @@ def _get_package_ids(client_cache, reference, package_ids): def cmd_copy(reference, user_channel, package_ids, client_cache, user_io, remote_manager, force=False): + """ + param package_ids: Falsey=do not copy binaries. True=All existing. []=list of ids + """ src_ref = ConanFileReference.loads(reference) short_paths = _prepare_sources(client_cache, user_io, remote_manager, src_ref) diff --git a/conans/client/command.py b/conans/client/command.py index 054acc24d04..2f71d129406 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -741,10 +741,9 @@ def copy(self, *args): """ parser = argparse.ArgumentParser(description=self.copy.__doc__, prog="conan copy") parser.add_argument("reference", default="", - help='package recipe reference' - 'e.g., MyPackage/1.2@user/channel') + help='package reference. e.g., MyPackage/1.2@user/channel') parser.add_argument("user_channel", default="", - help='Destination user/channel' + help='Destination user/channel. ' 'e.g., lasote/testing') parser.add_argument("--package", "-p", nargs=1, action=Extender, help='copy specified package ID') diff --git a/conans/test/command/copy_packages_test.py b/conans/test/command/copy_packages_test.py index 780305bab60..bb5426e7b27 100644 --- a/conans/test/command/copy_packages_test.py +++ b/conans/test/command/copy_packages_test.py @@ -1,38 +1,41 @@ import unittest from conans.test.utils.tools import TestClient import os -from conans.test.utils.cpp_test_files import cpp_hello_conan_files -from conans.paths import CONANFILE from conans.model.ref import ConanFileReference -from conans.util.files import rmdir class CopyPackagesTest(unittest.TestCase): def test_copy_command(self): client = TestClient() - self._export_some_packages(client) + conanfile = """from conans import ConanFile +class Pkg(ConanFile): + settings = "os" +""" + client.save({"conanfile.py": conanfile}) + client.run("export Hello0/0.1@lasote/stable") + client.run("install Hello0/0.1@lasote/stable -s os=Windows --build missing") + client.run("install Hello0/0.1@lasote/stable -s os=Linux --build missing") + client.run("install Hello0/0.1@lasote/stable -s os=Macos --build missing") + # Copy all packages - new_reference = ConanFileReference.loads("Hello0/0.1@pepe/testing") - client.run("copy Hello0/0.1@lasote/stable pepe/testing --all --force") - p1 = client.paths.packages(new_reference) - packages = os.listdir(p1) + client.run("copy Hello0/0.1@lasote/stable pepe/testing --all") + pkgdir = client.paths.packages(ConanFileReference.loads("Hello0/0.1@pepe/testing")) + packages = os.listdir(pkgdir) self.assertEquals(len(packages), 3) # Copy just one - rmdir(p1) - client.run("copy Hello0/0.1@lasote/stable pepe/testing -p %s --force" % packages[0]) - packages = os.listdir(p1) + client.run("copy Hello0/0.1@lasote/stable pepe/stable -p %s" % packages[0]) + pkgdir = client.paths.packages(ConanFileReference.loads("Hello0/0.1@pepe/stable")) + packages = os.listdir(pkgdir) self.assertEquals(len(packages), 1) - def _export_some_packages(self, client): - files = cpp_hello_conan_files("Hello0", "0.1") - # No build. - files[CONANFILE] = files[CONANFILE].replace("def build(self):", - "def build(self):\n return\n") - client.save(files) - client.run("export lasote/stable") - client.run("install Hello0/0.1@lasote/stable -s os=Windows --build missing") - client.run("install Hello0/0.1@lasote/stable -s os=Linux --build missing") - client.run("install Hello0/0.1@lasote/stable -s os=Linux -s compiler=gcc " - "-s compiler.version=4.6 -s compiler.libcxx=libstdc++ --build missing") + # Force + client.run("copy Hello0/0.1@lasote/stable pepe/stable -p %s --force" % packages[0]) + packages = os.listdir(pkgdir) + self.assertEquals(len(packages), 1) + + # Copy only recipe + client.run("copy Hello0/0.1@lasote/stable pepe/alpha", ignore_error=True) + pkgdir = client.paths.packages(ConanFileReference.loads("Hello0/0.1@pepe/alpha")) + self.assertFalse(os.path.exists(pkgdir)) diff --git a/conans/test/functional/path_limit_test.py b/conans/test/functional/path_limit_test.py index 3dc38db68ce..809627919b0 100644 --- a/conans/test/functional/path_limit_test.py +++ b/conans/test/functional/path_limit_test.py @@ -117,7 +117,7 @@ def package_copier_test(self): client.save(files) client.run("export lasote/channel") client.run("install lib/0.1@lasote/channel --build") - client.run("copy lib/0.1@lasote/channel memsharded/stable") + client.run("copy lib/0.1@lasote/channel memsharded/stable --all") client.run("search") self.assertIn("lib/0.1@lasote/channel", client.user_io.out) self.assertIn("lib/0.1@memsharded/stable", client.user_io.out) diff --git a/conans/test/integration/symlinks_test.py b/conans/test/integration/symlinks_test.py index 82feebc58c1..df51bbab583 100644 --- a/conans/test/integration/symlinks_test.py +++ b/conans/test/integration/symlinks_test.py @@ -85,7 +85,7 @@ def package_files_test(self): class TestConan(ConanFile): name = "Hello" version = "0.1" - + def package(self): self.copy("*", symlinks=True) """ @@ -122,7 +122,7 @@ def export_and_copy_test(self): client.run("export lasote/stable") client.run("install --build -f=conanfile.txt") - client.run("copy Hello/0.1@lasote/stable team/testing") + client.run("copy Hello/0.1@lasote/stable team/testing --all") conan_ref = ConanFileReference.loads("Hello/0.1@lasote/stable") team_ref = ConanFileReference.loads("Hello/0.1@team/testing") package_ref = PackageReference(conan_ref, From 80c3684aaf42ba1c05022017c479c178221898bd Mon Sep 17 00:00:00 2001 From: James Date: Mon, 13 Nov 2017 22:36:54 +0100 Subject: [PATCH 21/38] not remove zip if conan config install local-zip (#2021) * not remove zip if conan config install local-zip * fixed cpu_count --- conans/client/conf/config_installer.py | 7 ++++--- conans/test/command/config_install_test.py | 1 + conans/test/functional/cmake_test.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/conans/client/conf/config_installer.py b/conans/client/conf/config_installer.py index b31e8ef9f4f..9959caf4264 100644 --- a/conans/client/conf/config_installer.py +++ b/conans/client/conf/config_installer.py @@ -34,9 +34,10 @@ def _process_git_repo(repo_url, client_cache, output, runner, tmp_folder): _process_folder(tmp_folder, client_cache, output) -def _process_zip_file(zippath, client_cache, output, tmp_folder): +def _process_zip_file(zippath, client_cache, output, tmp_folder, remove=False): unzip(zippath, tmp_folder) - os.unlink(zippath) + if remove: + os.unlink(zippath) _process_folder(tmp_folder, client_cache, output) @@ -77,7 +78,7 @@ def _process_download(item, client_cache, output, tmp_folder): output.info("Trying to download %s" % item) zippath = os.path.join(tmp_folder, "config.zip") tools.download(item, zippath, out=output) - _process_zip_file(zippath, client_cache, output, tmp_folder) + _process_zip_file(zippath, client_cache, output, tmp_folder, remove=True) def configuration_install(item, client_cache, output, runner): diff --git a/conans/test/command/config_install_test.py b/conans/test/command/config_install_test.py index 983e90999c0..18b695677dd 100644 --- a/conans/test/command/config_install_test.py +++ b/conans/test/command/config_install_test.py @@ -131,6 +131,7 @@ def install_file_test(self): zippath = self._create_zip() self.client.run('config install "%s"' % zippath) self._check(zippath) + self.assertTrue(os.path.exists(zippath)) def test_without_profile_folder(self): shutil.rmtree(self.client.client_cache.profiles_path) diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 079c1fdff46..1167debcf07 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -383,7 +383,8 @@ def test_run_tests(self): conan_file.settings = settings cmake = CMake(conan_file) cmake.test() - self.assertEqual('cmake --build %s' % CMakeTest.scape('. --target RUN_TESTS'), conan_file.command) + self.assertIn('cmake --build %s' % CMakeTest.scape('. --target RUN_TESTS -- /m:%i' % cpu_count()), + conan_file.command) cmake.generator = "Ninja Makefiles" cmake.test() From b5d0c078af7036b467b5ba5eccc5b70591d72fe5 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 14 Nov 2017 08:29:43 +0100 Subject: [PATCH 22/38] deprecated old CMake helper methods (#2019) --- conans/client/build/cmake.py | 85 +++++----------------------- conans/model/conan_file.py | 2 - conans/test/functional/cmake_test.py | 30 ++-------- 3 files changed, 18 insertions(+), 99 deletions(-) diff --git a/conans/client/build/cmake.py b/conans/client/build/cmake.py index 2720209b704..04fbaa84885 100644 --- a/conans/client/build/cmake.py +++ b/conans/client/build/cmake.py @@ -7,7 +7,6 @@ from conans.client import defs_to_string, join_arguments from conans.errors import ConanException from conans.model.conan_file import ConanFile -from conans.model.settings import Settings from conans.util.env_reader import get_env from conans.util.files import mkdir from conans.tools import cpu_count, args_to_string @@ -15,22 +14,6 @@ from conans.util.log import logger from conans.util.config_parser import get_bool_from_text -# Deprecated in 0.22 -deprecated_conanfile_param_message = ''' -******************************* WARNING!!! ************************************ - -Do not pass 'self' to configure() nor build() methods, it is deprecated and will be removed. - -Instance CMake with the conanfile instance instead: - - cmake = CMake(self) - cmake.configure() # Optional args, defs, source_dir and build_dir parameters - cmake.build() # Optional args, build_dir and target - - -********************************************************************************** -''' - def _get_env_cmake_system_name(): env_system_name = get_env("CONAN_CMAKE_SYSTEM_NAME", "") @@ -39,7 +22,7 @@ def _get_env_cmake_system_name(): class CMake(object): - def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True, + def __init__(self, conanfile, generator=None, cmake_system_name=True, parallel=True, build_type=None, toolset=None): """ :param settings_or_conanfile: Conanfile instance (or settings for retro compatibility) @@ -51,18 +34,11 @@ def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True :param toolset: Toolset name to use (such as llvm-vs2014) or none for default one, applies only to certain generators (e.g. Visual Studio) """ - if isinstance(settings_or_conanfile, Settings): - self._settings = settings_or_conanfile - self._conanfile = None - self.configure = self._configure_old - self.build = self._build_old - elif isinstance(settings_or_conanfile, ConanFile): - self._settings = settings_or_conanfile.settings - self._conanfile = settings_or_conanfile - self.configure = self._configure_new - self.build = self._build_new - else: - raise ConanException("First parameter of CMake() has to be a ConanFile instance.") + if not isinstance(conanfile, ConanFile): + raise ConanException("First argument of CMake() has to be ConanFile. Use CMake(self)") + + self._settings = conanfile.settings + self._conanfile = conanfile self._os = self._settings.get_safe("os") self._compiler = self._settings.get_safe("compiler") @@ -103,19 +79,6 @@ def build_type(self, build_type): def flags(self): return defs_to_string(self.definitions) - @staticmethod - def options_cmd_line(options, option_upper=True, value_upper=True): - """ FIXME: this function seems weird, not tested, not used. - Probably should be deprecated - """ - result = [] - for option, value in options.values.as_list(): - if value is not None: - option = option.upper() if option_upper else option - value = value.upper() if value_upper else value - result.append("-D%s=%s" % (option, value)) - return ' '.join(result) - def _generator(self): if not self._compiler or not self._compiler_version or not self._arch: @@ -296,14 +259,14 @@ def _get_cmake_definitions(self): # Shared library try: ret["BUILD_SHARED_LIBS"] = "ON" if self._conanfile.options.shared else "OFF" - except: + except ConanException: pass # Install to package folder try: if self._conanfile.package_folder: ret["CMAKE_INSTALL_PREFIX"] = self._conanfile.package_folder - except: + except AttributeError: pass if self._os == "Windows" and self._compiler == "Visual Studio": @@ -313,17 +276,7 @@ def _get_cmake_definitions(self): ret["CONAN_C_FLAGS"] = "/MP%s" % cpus return ret - def _configure_old(self, conanfile, args=None, defs=None, source_dir=None, build_dir=None): - """Deprecated in 0.22""" - if not isinstance(conanfile, ConanFile): - raise ConanException(deprecated_conanfile_param_message) - self._conanfile = conanfile - self._conanfile.output.warn(deprecated_conanfile_param_message) - return self._configure_new(args=args, defs=defs, source_dir=source_dir, build_dir=build_dir) - - def _configure_new(self, args=None, defs=None, source_dir=None, build_dir=None): - if isinstance(args, ConanFile): - raise ConanException(deprecated_conanfile_param_message) + def configure(self, args=None, defs=None, source_dir=None, build_dir=None): args = args or [] defs = defs or {} source_dir = source_dir or self._conanfile.source_folder @@ -343,17 +296,7 @@ def _configure_new(self, args=None, defs=None, source_dir=None, build_dir=None): else: self._conanfile.run(command) - def _build_old(self, conanfile, args=None, build_dir=None, target=None): - """Deprecated in 0.22""" - if not isinstance(conanfile, ConanFile): - raise ConanException(deprecated_conanfile_param_message) - self._conanfile = conanfile - self._conanfile.output.warn(deprecated_conanfile_param_message) - return self._build_new(args=args, build_dir=build_dir, target=target) - - def _build_new(self, args=None, build_dir=None, target=None): - if isinstance(args, ConanFile): - raise ConanException(deprecated_conanfile_param_message) + def build(self, args=None, build_dir=None, target=None): args = args or [] build_dir = build_dir or self.build_dir or self._conanfile.build_folder if target is not None: @@ -382,21 +325,19 @@ def install(self, args=None, build_dir=None): if not self.definitions.get("CMAKE_INSTALL_PREFIX"): raise ConanException("CMAKE_INSTALL_PREFIX not defined for 'cmake.install()'\n" "Make sure 'package_folder' is defined") - self._build_new(args=args, build_dir=build_dir, target="install") + self.build(args=args, build_dir=build_dir, target="install") def test(self, args=None, build_dir=None, target=None): - if isinstance(args, ConanFile): - raise ConanException(deprecated_conanfile_param_message) if not target: target = "RUN_TESTS" if self.is_multi_configuration else "test" - self._build_new(args=args, build_dir=build_dir, target=target) + self.build(args=args, build_dir=build_dir, target=target) @property def verbose(self): try: verbose = self.definitions["CMAKE_VERBOSE_MAKEFILE"] return get_bool_from_text(str(verbose)) - except: + except KeyError: return False @verbose.setter diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 3eab88112ee..c676d1875f3 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -1,5 +1,3 @@ -import copy - from conans.model.options import Options, PackageOptions, OptionsValues from conans.model.requires import Requirements from conans.model.build_info import DepsCppInfo diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 1167debcf07..f799b466703 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -15,6 +15,8 @@ from conans.tools import cpu_count from conans.util.files import save from conans.test.utils.test_files import temp_folder +from conans.model.options import Options, PackageOptions +from conans.errors import ConanException class CMakeTest(unittest.TestCase): @@ -225,33 +227,10 @@ def test_deprecated_behaviour(self): conanfile to configure/build/test""" settings = Settings.loads(default_settings_yml) settings.os = "Windows" - settings.compiler = "Visual Studio" - settings.compiler.version = "12" - settings.arch = "x86" - settings.os = "Windows" - - dot_dir = "." if sys.platform == 'win32' else "'.'" - - cross = "-DCMAKE_SYSTEM_NAME=\"Windows\" " if platform.system() != "Windows" else "" - conan_file = ConanFileMock() conan_file.settings = settings - cmake = CMake(settings) - cmake.configure(conan_file) - cores = '-DCONAN_CXX_FLAGS="/MP{0}" -DCONAN_C_FLAGS="/MP{0}" '.format(tools.cpu_count()) - self.assertEqual('cd {0} && cmake -G "Visual Studio 12 2013" {1}-DCONAN_EXPORTED="1" ' - '-DCONAN_COMPILER="Visual Studio" -DCONAN_COMPILER_VERSION="12" {2}' - '-Wno-dev {0}'.format(dot_dir, cross, cores), - conan_file.command) - - cmake.build(conan_file) - self.assertEqual('cmake --build %s %s' - % (dot_dir, (CMakeTest.scape('-- /m:%i' % cpu_count()))), conan_file.command) - cmake.test() - self.assertEqual('cmake --build %s %s %s' - % (dot_dir, CMakeTest.scape('--target RUN_TESTS'), - (CMakeTest.scape('-- /m:%i' % cpu_count()))), - conan_file.command) + with self.assertRaises(ConanException): + CMake(settings) def convenient_functions_test(self): settings = Settings.loads(default_settings_yml) @@ -515,6 +494,7 @@ def __init__(self, shared=None): self.conanfile_directory = "." self.source_folder = self.build_folder = "." self.settings = None + self.options = Options(PackageOptions.loads("")) self.deps_cpp_info = namedtuple("deps_cpp_info", "sysroot")("/path/to/sysroot") self.output = TestBufferConanOutput() if shared is not None: From 0fa727bdd802375ddc4d61c6a35d2698f2a7cfe4 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Tue, 14 Nov 2017 16:08:49 +0100 Subject: [PATCH 23/38] Autotools autoloading pkg_config_path (#1950) * Autotools autoloading pkg_config_path * Fixed test --- conans/client/build/autotools_environment.py | 22 ++++++++--- .../functional/autotools_configure_test.py | 37 +++++++++++++++++++ conans/test/utils/conanfile.py | 1 + 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/conans/client/build/autotools_environment.py b/conans/client/build/autotools_environment.py index 0dc8ef0a6e5..b77debb6a6b 100644 --- a/conans/client/build/autotools_environment.py +++ b/conans/client/build/autotools_environment.py @@ -110,8 +110,10 @@ def _get_host_build_target_flags(self, arch_detected, os_detected): return build, host, None - def configure(self, configure_dir=None, args=None, build=None, host=None, target=None): + def configure(self, configure_dir=None, args=None, build=None, host=None, target=None, + pkg_config_paths=None): """ + :param pkg_config_paths: Optional paths to locate the *.pc files :param configure_dir: Absolute or relative path to the configure script :param args: Optional arguments to pass to configure. :param build: In which system the program will be built. "False" skips the --build flag @@ -147,9 +149,18 @@ def configure(self, configure_dir=None, args=None, build=None, host=None, target if target or auto_target: # User specified value or automatic triplet_args.append("--target=%s" % (target or auto_target)) - with environment_append(self.vars): - self._conanfile.run("%s/configure %s %s" - % (configure_dir, args_to_string(args), " ".join(triplet_args))) + if pkg_config_paths: + pkg_env = {"PKG_CONFIG_PATH": os.pathsep.join(pkg_config_paths)} + else: + # If we are using pkg_config generator automate the pcs location, otherwise it could + # read wrong files + pkg_env = {"PKG_CONFIG_PATH": self._conanfile.build_folder} \ + if "pkg_config" in self._conanfile.generators else {} + + with environment_append(pkg_env): + with environment_append(self.vars): + self._conanfile.run("%s/configure %s %s" + % (configure_dir, args_to_string(args), " ".join(triplet_args))) def make(self, args="", make_program=None): make_program = os.getenv("CONAN_MAKE_PROGRAM") or make_program or "make" @@ -177,7 +188,8 @@ def _configure_flags(self): if self._build_type == "Debug": ret.append("-g") # default debug information elif self._build_type == "Release" and self._compiler == "gcc": - ret.append("-s") # Remove all symbol table and relocation information from the executable. + # Remove all symbol table and relocation information from the executable. + ret.append("-s") if self._sysroot_flag: ret.append(self._sysroot_flag) return ret diff --git a/conans/test/functional/autotools_configure_test.py b/conans/test/functional/autotools_configure_test.py index 7ca1b9df6b0..c20f7c22f81 100644 --- a/conans/test/functional/autotools_configure_test.py +++ b/conans/test/functional/autotools_configure_test.py @@ -4,8 +4,10 @@ from conans.client.build.autotools_environment import AutoToolsBuildEnvironment from conans import tools from conans.client.tools.oss import cpu_count +from conans.paths import CONANFILE from conans.test.utils.conanfile import MockConanfile, MockSettings from conans.test.util.tools_test import RunnerMock +from conans.test.utils.tools import TestClient class AutoToolsConfigureTest(unittest.TestCase): @@ -349,6 +351,7 @@ def get_values(this_os, this_arch, setting_os, setting_arch): self.assertFalse(target) build, host, target = get_values("Darwin", "x86_64", "Android", "armv7hf") + self.assertEquals(build, "x86_64-apple-darwin") self.assertEquals(host, "arm-linux-androideabi") @@ -368,6 +371,40 @@ def get_values(this_os, this_arch, setting_os, setting_arch): self.assertEquals(build, "x86_64-apple-darwin") self.assertEquals(host, "arm-apple-darwin") + + def test_pkg_config_paths(self): + if platform.system() == "Windows": + return + client = TestClient() + conanfile = """ +from conans import ConanFile, tools, AutoToolsBuildEnvironment + +class HelloConan(ConanFile): + name = "Hello" + version = "1.2.1" + generators = %s + + def build(self): + tools.save("configure", "printenv") + self.run("chmod +x configure") + autot = AutoToolsBuildEnvironment(self) + autot.configure(%s) + +""" + + client.save({CONANFILE: conanfile % ("'txt'", "")}) + client.run("create conan/testing") + self.assertNotIn("PKG_CONFIG_PATH=", client.out) + + client.save({CONANFILE: conanfile % ("'pkg_config'", "")}) + client.run("create conan/testing") + self.assertIn("PKG_CONFIG_PATH=%s" % client.client_cache.conan_folder, client.out) + + client.save({CONANFILE: conanfile % ("'pkg_config'", + "pkg_config_paths=['/tmp/hello', '/tmp/foo']")}) + client.run("create conan/testing") + self.assertIn("PKG_CONFIG_PATH=/tmp/hello:/tmp/foo", client.out) + def cross_build_command_test(self): runner = RunnerMock() conanfile = MockConanfile(MockSettings({}), runner) diff --git a/conans/test/utils/conanfile.py b/conans/test/utils/conanfile.py index d283259b805..b769c67b36e 100644 --- a/conans/test/utils/conanfile.py +++ b/conans/test/utils/conanfile.py @@ -28,6 +28,7 @@ def __init__(self, settings, runner=None): self.deps_cpp_info = MockDepsCppInfo() self.settings = settings self.runner = runner + self.generators = [] def run(self, *args): if self.runner: From 8fb36478ee2a481a1a20a846e188f18ece4f2d02 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 14 Nov 2017 16:53:50 +0100 Subject: [PATCH 24/38] more refactors of cmd (#2023) --- .../client/{package_tester.py => cmd/test.py} | 0 conans/client/{ => cmd}/uploader.py | 53 +++++++++++++------ conans/client/command.py | 2 +- conans/client/conan_api.py | 19 +++---- conans/client/manager.py | 26 --------- conans/test/functional/package_tester_test.py | 2 +- 6 files changed, 48 insertions(+), 54 deletions(-) rename conans/client/{package_tester.py => cmd/test.py} (100%) rename conans/client/{ => cmd}/uploader.py (61%) diff --git a/conans/client/package_tester.py b/conans/client/cmd/test.py similarity index 100% rename from conans/client/package_tester.py rename to conans/client/cmd/test.py diff --git a/conans/client/uploader.py b/conans/client/cmd/uploader.py similarity index 61% rename from conans/client/uploader.py rename to conans/client/cmd/uploader.py index e9f54c4b5d8..edafcdbe911 100644 --- a/conans/client/uploader.py +++ b/conans/client/cmd/uploader.py @@ -5,9 +5,10 @@ from conans.model.ref import PackageReference, ConanFileReference from conans.util.log import logger from conans.client.loader_parse import load_conanfile_class +from conans.client.proxy import ConanProxy -def is_a_reference(ref): +def _is_a_reference(ref): try: ConanFileReference.loads(ref) return "*" not in ref # If is a pattern, it is not a reference @@ -16,20 +17,42 @@ def is_a_reference(ref): return False -class ConanUploader(object): +class CmdUpload(object): - def __init__(self, paths, user_io, remote_proxy, search_manager): - self._paths = paths + def __init__(self, client_cache, user_io, remote_manager, search_manager, remote): + self._client_cache = client_cache self._user_io = user_io - self._remote_proxy = remote_proxy + self._remote_proxy = ConanProxy(self._client_cache, self._user_io, remote_manager, remote) self._search_manager = search_manager - def upload(self, pattern, force=False, all_packages=False, confirm=False, - retry=None, retry_wait=None, skip_upload=False, integrity_check=False): + def upload(self, conan_reference_or_pattern, package_id=None, all_packages=None, + force=False, confirm=False, retry=0, retry_wait=0, skip_upload=False, + integrity_check=False): + """If package_id is provided, conan_reference_or_pattern is a ConanFileReference""" + if package_id and not _is_a_reference(conan_reference_or_pattern): + raise ConanException("-p parameter only allowed with a valid recipe reference, " + "not with a pattern") + t1 = time.time() + if package_id: # Upload package + ref = ConanFileReference.loads(conan_reference_or_pattern) + self._check_reference(ref) + self.upload_package(PackageReference(ref, package_id), retry=retry, + retry_wait=retry_wait, skip_upload=skip_upload, + integrity_check=integrity_check) + else: # Upload conans + self._run_upload(conan_reference_or_pattern, all_packages=all_packages, + force=force, confirm=confirm, + retry=retry, retry_wait=retry_wait, skip_upload=skip_upload, + integrity_check=integrity_check) + + logger.debug("====> Time manager upload: %f" % (time.time() - t1)) + + def _run_upload(self, pattern, force=False, all_packages=False, confirm=False, + retry=None, retry_wait=None, skip_upload=False, integrity_check=False): """Upload all the recipes matching 'pattern'""" - if is_a_reference(pattern): + if _is_a_reference(pattern): ref = ConanFileReference.loads(pattern) - export_path = self._paths.export(ref) + export_path = self._client_cache.export(ref) if not os.path.exists(export_path): raise NotFoundException("There is no local conanfile exported as %s" % str(ref)) @@ -58,16 +81,16 @@ def _upload(self, conan_ref, force, all_packages, retry, retry_wait, skip_upload self._user_io.out.info("Uploading %s" % str(conan_ref)) self._remote_proxy.upload_recipe(conan_ref, retry, retry_wait, skip_upload) if all_packages: - self.check_reference(conan_ref) + self._check_reference(conan_ref) - for index, package_id in enumerate(self._paths.conan_packages(conan_ref)): - total = len(self._paths.conan_packages(conan_ref)) + for index, package_id in enumerate(self._client_cache.conan_packages(conan_ref)): + total = len(self._client_cache.conan_packages(conan_ref)) self.upload_package(PackageReference(conan_ref, package_id), index + 1, total, retry, retry_wait, skip_upload, integrity_check) - def check_reference(self, conan_reference): + def _check_reference(self, conan_reference): try: - conanfile_path = self._paths.conanfile(conan_reference) + conanfile_path = self._client_cache.conanfile(conan_reference) conan_file = load_conanfile_class(conanfile_path) except NotFoundException: raise NotFoundException("There is no local conanfile exported as %s" @@ -95,7 +118,7 @@ def _check_recipe_date(self, conan_ref): except NotFoundException: return # First upload - local_manifest = self._paths.load_manifest(conan_ref) + local_manifest = self._client_cache.load_manifest(conan_ref) if (remote_recipe_manifest != local_manifest and remote_recipe_manifest.time > local_manifest.time): diff --git a/conans/client/command.py b/conans/client/command.py index 2f71d129406..9a87756f213 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -893,7 +893,7 @@ def upload(self, *args): args = parser.parse_args(*args) return self._conan.upload(pattern=args.pattern, package=args.package, remote=args.remote, - all=args.all, + all_packages=args.all, force=args.force, confirm=args.confirm, retry=args.retry, retry_wait=args.retry_wait, skip_upload=args.skip_upload, integrity_check=args.check) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 09c365e98e6..9e967095fbb 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -21,7 +21,7 @@ from conans.client.rest.version_checker import VersionCheckerRequester from conans.client.runner import ConanRunner from conans.client.store.localdb import LocalDB -from conans.client.package_tester import PackageTester +from conans.client.cmd.test import PackageTester from conans.client.userio import UserIO from conans.errors import ConanException from conans.model.env_info import EnvValues @@ -39,7 +39,7 @@ from conans.client.loader_parse import load_conanfile_class from conans.client import settings_preprocessor from conans.tools import set_global_instances -from conans.client.uploader import is_a_reference +from conans.client.cmd.uploader import CmdUpload default_manifest_folder = '.conan_manifests' @@ -716,17 +716,14 @@ def search_packages(self, reference, query=None, remote=None, outdated=False): return ret @api_method - def upload(self, pattern, package=None, remote=None, all=False, force=False, confirm=False, - retry=2, retry_wait=5, skip_upload=False, integrity_check=False): + def upload(self, pattern, package=None, remote=None, all_packages=False, force=False, + confirm=False, retry=2, retry_wait=5, skip_upload=False, integrity_check=False): """ Uploads a package recipe and the generated binary packages to a specified remote """ - if package and not is_a_reference(pattern): - raise ConanException("-p parameter only allowed with a valid recipe reference, " - "not with a pattern") - - self._manager.upload(pattern, package, remote, all_packages=all, force=force, - confirm=confirm, retry=retry, retry_wait=retry_wait, - skip_upload=skip_upload, integrity_check=integrity_check) + uploader = CmdUpload(self._client_cache, self._user_io, self._manager._remote_manager, + self._manager._search_manager, remote) + return uploader.upload(pattern, package, all_packages, force, confirm, retry, + retry_wait, skip_upload, integrity_check) @api_method def remote_list(self): diff --git a/conans/client/manager.py b/conans/client/manager.py index 03e269605aa..d620b5cfbca 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -1,6 +1,5 @@ import fnmatch import os -import time from collections import OrderedDict, Counter from conans.client import packager @@ -23,7 +22,6 @@ from conans.client.remover import ConanRemover from conans.client.require_resolver import RequireResolver from conans.client.source import config_source_local -from conans.client.uploader import ConanUploader from conans.client.userio import UserIO from conans.errors import NotFoundException, ConanException, conanfile_exception_formatter from conans.model.manifest import FileTreeManifest @@ -491,30 +489,6 @@ def build(self, conanfile_path, source_folder, build_folder, package_folder, ins trace = traceback.format_exc().split('\n') raise ConanException("Unable to build it successfully\n%s" % '\n'.join(trace[3:])) - def upload(self, conan_reference_or_pattern, package_id=None, remote=None, all_packages=None, - force=False, confirm=False, retry=0, retry_wait=0, skip_upload=False, - integrity_check=False): - """If package_id is provided, conan_reference_or_pattern is a ConanFileReference""" - t1 = time.time() - remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, - remote) - uploader = ConanUploader(self._client_cache, self._user_io, remote_proxy, - self._search_manager) - - if package_id: # Upload package - ref = ConanFileReference.loads(conan_reference_or_pattern) - uploader.check_reference(ref) - uploader.upload_package(PackageReference(ref, package_id), retry=retry, - retry_wait=retry_wait, skip_upload=skip_upload, - integrity_check=integrity_check) - else: # Upload conans - uploader.upload(conan_reference_or_pattern, all_packages=all_packages, - force=force, confirm=confirm, - retry=retry, retry_wait=retry_wait, skip_upload=skip_upload, - integrity_check=integrity_check) - - logger.debug("====> Time manager upload: %f" % (time.time() - t1)) - def _get_search_adapter(self, remote): if remote: remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) diff --git a/conans/test/functional/package_tester_test.py b/conans/test/functional/package_tester_test.py index f8a2f42cfc8..7ccc2022974 100644 --- a/conans/test/functional/package_tester_test.py +++ b/conans/test/functional/package_tester_test.py @@ -1,6 +1,6 @@ import unittest -from conans.client.package_tester import PackageTester +from conans.client.cmd.test import PackageTester from conans.errors import ConanException from conans.model.ref import ConanFileReference from conans.model.requires import Requirement From 412ab90de4c7f30018f1cad5dfbeca2409d9e892 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Wed, 15 Nov 2017 06:16:04 +0700 Subject: [PATCH 25/38] - tools.PkgConfig helper (#1814) * - tools.PkgConfig helper * - use frozenset as order is not important * - lazy implementation * - add test case to demonstrate how to override prefix --- conans/client/tools/__init__.py | 2 + conans/client/tools/pkg_config.py | 91 +++++++++++++++++++++++++++++ conans/test/util/pkg_config_test.py | 73 +++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 conans/client/tools/pkg_config.py create mode 100644 conans/test/util/pkg_config_test.py diff --git a/conans/client/tools/__init__.py b/conans/client/tools/__init__.py index a36eae84c0f..c1f321c3822 100644 --- a/conans/client/tools/__init__.py +++ b/conans/client/tools/__init__.py @@ -11,3 +11,5 @@ from .system_pm import * # noinspection PyUnresolvedReferences from .win import * +# noinspection PyUnresolvedReferences +from .pkg_config import * \ No newline at end of file diff --git a/conans/client/tools/pkg_config.py b/conans/client/tools/pkg_config.py new file mode 100644 index 00000000000..00ec7e0b841 --- /dev/null +++ b/conans/client/tools/pkg_config.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import subprocess + + +class PkgConfig(object): + @staticmethod + def _cmd_output(command): + return subprocess.check_output(command).decode().strip() + + def __init__(self, library, pkg_config_executable='pkg-config', static=False, msvc_syntax=False, variables=None): + """ + :param library: library (package) name, such as libastral + :param pkg_config_executable: specify custom pkg-config executable (e.g. for cross-compilation) + :param static: output libraries suitable for static linking (adds --static to pkg-config command line) + :param msvc_syntax: MSVC compatibility (adds --msvc-syntax to pkg-config command line) + :param variables: dictionary of pkg-config variables (passed as --define-variable=VARIABLENAME=VARIABLEVALUE) + """ + self.library = library + self.pkg_config_executable = pkg_config_executable + self.static = static + self.msvc_syntax = msvc_syntax + self.define_variables = variables + + self._variables = dict() + self.info = dict() + + def _parse_output(self, option): + command = [self.pkg_config_executable, '--' + option, self.library] + if self.static: + command.append('--static') + if self.msvc_syntax: + command.append('--msvc-syntax') + if self.define_variables: + for name, value in self.define_variables.items(): + command.append('--define-variable=%s=%s' % (name, value)) + return self._cmd_output(command) + + def _get_option(self, option): + if not option in self.info: + self.info[option] = self._parse_output(option).split() + return self.info[option] + + @property + def cflags(self): + return self._get_option('cflags') + + @property + def cflags_only_I(self): + return self._get_option('cflags-only-I') + + @property + def cflags_only_other(self): + return self._get_option('cflags-only-other') + + @property + def libs(self): + return self._get_option('libs') + + @property + def libs_only_L(self): + return self._get_option('libs-only-L') + + @property + def libs_only_l(self): + return self._get_option('libs-only-l') + + @property + def libs_only_other(self): + return self._get_option('libs-only-other') + + @property + def provides(self): + return self._get_option('print-provides') + + @property + def requires(self): + return self._get_option('print-requires') + + @property + def requires_private(self): + return self._get_option('print-requires-private') + + @property + def variables(self): + if not self._variables: + variable_names = self._parse_output('print-variables').split() + for name in variable_names: + self._variables[name] = self._parse_output('variable=%s' % name) + return self._variables diff --git a/conans/test/util/pkg_config_test.py b/conans/test/util/pkg_config_test.py new file mode 100644 index 00000000000..3f51ffbb266 --- /dev/null +++ b/conans/test/util/pkg_config_test.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import platform +import os +from nose.plugins.attrib import attr +from conans.tools import PkgConfig, environment_append +from conans.test.utils.test_files import temp_folder + +libastral_pc = """ +PC FILE EXAMPLE: + +prefix=/usr/local +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: libastral +Description: Interface library for Astral data flows +Version: 6.6.6 +Libs: -L${libdir}/libastral -lastral -Wl,--whole-archive +Cflags: -I${includedir}/libastral -D_USE_LIBASTRAL +""" + + +@attr("unix") +class PkgConfigTest(unittest.TestCase): + def test_pc(self): + if platform.system() == "Windows": + return + tmp_dir = temp_folder() + filename = os.path.join(tmp_dir, 'libastral.pc') + with open(filename, 'w') as f: + f.write(libastral_pc) + with environment_append({'PKG_CONFIG_PATH': tmp_dir}): + pkg_config = PkgConfig("libastral") + + self.assertEquals(frozenset(pkg_config.cflags), frozenset(['-D_USE_LIBASTRAL', '-I/usr/local/include/libastral'])) + self.assertEquals(frozenset(pkg_config.cflags_only_I), frozenset(['-I/usr/local/include/libastral'])) + self.assertEquals(frozenset(pkg_config.cflags_only_other), frozenset(['-D_USE_LIBASTRAL'])) + + self.assertEquals(frozenset(pkg_config.libs), frozenset(['-L/usr/local/lib/libastral', '-lastral', '-Wl,--whole-archive'])) + self.assertEquals(frozenset(pkg_config.libs_only_L), frozenset(['-L/usr/local/lib/libastral'])) + self.assertEquals(frozenset(pkg_config.libs_only_l), frozenset(['-lastral',])) + self.assertEquals(frozenset(pkg_config.libs_only_other), frozenset(['-Wl,--whole-archive'])) + + self.assertEquals(pkg_config.variables['prefix'], '/usr/local') + os.unlink(filename) + + def test_define_prefix(self): + if platform.system() == "Windows": + return + tmp_dir = temp_folder() + filename = os.path.join(tmp_dir, 'libastral.pc') + with open(filename, 'w') as f: + f.write(libastral_pc) + with environment_append({'PKG_CONFIG_PATH': tmp_dir}): + pkg_config = PkgConfig("libastral", variables={'prefix': '/home/conan'}) + + self.assertEquals(frozenset(pkg_config.cflags), + frozenset(['-D_USE_LIBASTRAL', '-I/home/conan/include/libastral'])) + self.assertEquals(frozenset(pkg_config.cflags_only_I), frozenset(['-I/home/conan/include/libastral'])) + self.assertEquals(frozenset(pkg_config.cflags_only_other), frozenset(['-D_USE_LIBASTRAL'])) + + self.assertEquals(frozenset(pkg_config.libs), + frozenset(['-L/home/conan/lib/libastral', '-lastral', '-Wl,--whole-archive'])) + self.assertEquals(frozenset(pkg_config.libs_only_L), frozenset(['-L/home/conan/lib/libastral'])) + self.assertEquals(frozenset(pkg_config.libs_only_l), frozenset(['-lastral', ])) + self.assertEquals(frozenset(pkg_config.libs_only_other), frozenset(['-Wl,--whole-archive'])) + + self.assertEquals(pkg_config.variables['prefix'], '/home/conan') + os.unlink(filename) From feaddfe185f45a3572a5ad7aa2d9bbb4b7a6315b Mon Sep 17 00:00:00 2001 From: James Date: Wed, 15 Nov 2017 11:56:19 +0100 Subject: [PATCH 26/38] fixed type in api download (#2027) --- conans/client/command.py | 3 +-- conans/client/conan_api.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/conans/client/command.py b/conans/client/command.py index 9a87756f213..8f158371358 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -268,9 +268,8 @@ def download(self, *args): action=OnceArgument) args = parser.parse_args(*args) - reference = ConanFileReference.loads(args.reference) - return self._conan.download(reference=reference, package=args.package, remote=args.remote) + return self._conan.download(reference=args.reference, package=args.package, remote=args.remote) def install(self, *args): """Installs the requirements specified in a conanfile (.py or .txt). diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 9e967095fbb..79bd2200b47 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -433,10 +433,8 @@ def export_pkg(self, path, name, channel, source_folder=None, build_folder=None, @api_method def download(self, reference, remote=None, package=None): # Install packages without settings (fixed ids or all) - if not reference or not isinstance(reference, ConanFileReference): - raise ConanException("Invalid package recipe reference. " - "e.g., MyPackage/1.2@user/channel") - self._manager.download(reference, package, remote=remote) + conan_ref = ConanFileReference.loads(reference) + self._manager.download(conan_ref, package, remote=remote) @api_method def install_reference(self, reference, settings=None, options=None, env=None, scope=None, From 50fdb825c03635f9bd4bf21935cd0a2a6720d5e6 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 15 Nov 2017 21:27:52 +0100 Subject: [PATCH 27/38] complete sources not necessary if not exports_sources (#2022) * complete sources not necessary if not exports_sources * fixed broken test --- conans/client/cmd/copy.py | 3 ++- conans/client/proxy.py | 11 +++++++--- conans/test/command/upload_test.py | 20 +++++++++++++++++ .../test/integration/export_sources_test.py | 7 +++++- conans/test/utils/tools.py | 22 ++++++++++--------- 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/conans/client/cmd/copy.py b/conans/client/cmd/copy.py index bee0bf80f38..b01745ed44b 100644 --- a/conans/client/cmd/copy.py +++ b/conans/client/cmd/copy.py @@ -11,7 +11,8 @@ def _prepare_sources(client_cache, user_io, remote_manager, reference): remote_proxy = ConanProxy(client_cache, user_io, remote_manager, None) conan_file_path = client_cache.conanfile(reference) conanfile = load_conanfile_class(conan_file_path) - remote_proxy.complete_recipe_sources(reference, short_paths=conanfile.short_paths) + remote_proxy.complete_recipe_sources(conanfile, reference, + short_paths=conanfile.short_paths) return conanfile.short_paths diff --git a/conans/client/proxy.py b/conans/client/proxy.py index 7185219b4e3..00ebc3c723e 100644 --- a/conans/client/proxy.py +++ b/conans/client/proxy.py @@ -1,6 +1,6 @@ from conans.client.local_file_getter import get_path from conans.client.output import ScopedOutput -from conans.util.files import rmdir +from conans.util.files import rmdir, mkdir from conans.model.ref import PackageReference from conans.errors import (ConanException, ConanConnectionError, ConanOutdatedClient, NotFoundException) @@ -234,8 +234,12 @@ def _retrieve_from_remote(remote): raise ConanException("No remote defined") - def complete_recipe_sources(self, conan_reference, force_complete=True, short_paths=False): + def complete_recipe_sources(self, conanfile, conan_reference, force_complete=True, short_paths=False): sources_folder = self._client_cache.export_sources(conan_reference, short_paths) + if not hasattr(conanfile, "exports_sources"): + mkdir(sources_folder) + return None + ignore_deleted_file = None if not os.path.exists(sources_folder): # If not path to sources exists, we have a problem, at least an empty folder @@ -261,7 +265,8 @@ def upload_recipe(self, conan_reference, retry, retry_wait, skip_upload): """ conan_file_path = self._client_cache.conanfile(conan_reference) conanfile = load_conanfile_class(conan_file_path) - ignore_deleted_file = self.complete_recipe_sources(conan_reference, force_complete=False, + ignore_deleted_file = self.complete_recipe_sources(conanfile, conan_reference, + force_complete=False, short_paths=conanfile.short_paths) remote, ref_remote = self._get_remote(conan_reference) diff --git a/conans/test/command/upload_test.py b/conans/test/command/upload_test.py index 3fcc9974aff..e73a76f3fb8 100644 --- a/conans/test/command/upload_test.py +++ b/conans/test/command/upload_test.py @@ -221,3 +221,23 @@ def skip_upload_test(self): # now it should be on the server client.run("search -r default") self.assertIn("Hello0/1.2.1@frodo/stable", client.user_io.out) + + def upload_without_sources_test(self): + client = self._client() + conanfile = """from conans import ConanFile +class Pkg(ConanFile): + pass +""" + client.save({"conanfile.py": conanfile}) + client.run("create Pkg/0.1@user/testing") + client.run("upload * --all --confirm") + client2 = self._client() + client2.run("install Pkg/0.1@user/testing") + client2.run("remote remove default") + server2 = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")], + users={"lasote": "mypass"}) + client2.users = {"server2": [("lasote", "mypass")]} + client2.update_servers({"server2": server2}) + client2.run("upload * --all --confirm -r=server2") + self.assertIn("Uploading conanfile.py", client2.out) + self.assertIn("Uploading conan_package.tgz", client2.out) diff --git a/conans/test/integration/export_sources_test.py b/conans/test/integration/export_sources_test.py index 537944455cb..36e1ae5fa72 100644 --- a/conans/test/integration/export_sources_test.py +++ b/conans/test/integration/export_sources_test.py @@ -166,7 +166,12 @@ def _check_export_installed_folder(self, mode, reuploaded=False, updated=False): expected_exports.append("license.txt") self.assertEqual(scan_folder(self.export_folder), sorted(expected_exports)) - self.assertFalse(os.path.exists(self.export_sources_folder)) + if reuploaded and mode == "exports": + # In this mode, we know the sources are not required + # So the local folder is created, but empty + self.assertTrue(os.path.exists(self.export_sources_folder)) + else: + self.assertFalse(os.path.exists(self.export_sources_folder)) def _check_export_uploaded_folder(self, mode, export_folder=None, export_src_folder=None): if mode == "exports_sources": diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py index c01f2fbfd08..07e28b88b25 100644 --- a/conans/test/utils/tools.py +++ b/conans/test/utils/tools.py @@ -307,7 +307,6 @@ def __init__(self, base_folder=None, current_folder=None, self.all_output = "" # For debugging purpose, append all the run outputs self.users = users or {"default": [(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]} - self.servers = servers or {} self.client_version = Version(str(client_version)) self.min_server_compatible_version = Version(str(min_server_compatible_version)) @@ -323,16 +322,9 @@ def __init__(self, base_folder=None, current_folder=None, self.requester_class = requester_class self.conan_runner = runner + self.update_servers(servers) self.init_dynamic_vars() - save(self.client_cache.registry, "") - registry = RemoteRegistry(self.client_cache.registry, TestBufferConanOutput()) - for name, server in self.servers.items(): - if isinstance(server, TestServer): - registry.add(name, server.fake_url) - else: - registry.add(name, server) - logger.debug("Client storage = %s" % self.storage_folder) self.current_folder = current_folder or temp_folder(path_with_spaces) @@ -343,6 +335,16 @@ def __init__(self, base_folder=None, current_folder=None, profile.settings["compiler.version"] = "14" save(self.client_cache.default_profile_path, profile.dumps()) + def update_servers(self, servers): + self.servers = servers or {} + save(self.client_cache.registry, "") + registry = RemoteRegistry(self.client_cache.registry, TestBufferConanOutput()) + for name, server in self.servers.items(): + if isinstance(server, TestServer): + registry.add(name, server.fake_url) + else: + registry.add(name, server) + @property def paths(self): return self.client_cache @@ -419,7 +421,7 @@ def run(self, command_line, user_io=None, ignore_error=False): """ self.init_dynamic_vars(user_io) conan = Conan(self.client_cache, self.user_io, self.runner, self.remote_manager, - self.search_manager, settings_preprocessor) + self.search_manager, settings_preprocessor) outputer = CommandOutputer(self.user_io, self.client_cache) command = Command(conan, self.client_cache, self.user_io, outputer) args = shlex.split(command_line) From dd7c792be344866c92b191c69482a6343e6d9e09 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Thu, 16 Nov 2017 10:05:01 +0100 Subject: [PATCH 28/38] Feature/toolsets (#2024) * Toolset subsetting and not limited vcvars * Toolsets in msbuild * pass toolset * Fixed toolset param * Added more toolsets * Added test --- conans/client/build/cmake.py | 10 +++-- conans/client/conf/__init__.py | 1 + conans/client/tools/win.py | 24 ++++++---- conans/model/info.py | 27 +++++++++++ conans/test/functional/cmake_test.py | 17 ++++++- conans/test/integration/package_id_test.py | 52 +++++++++++++++++++++- conans/test/util/build_sln_command_test.py | 7 +++ conans/test/util/tools_test.py | 20 +++++---- 8 files changed, 136 insertions(+), 22 deletions(-) diff --git a/conans/client/build/cmake.py b/conans/client/build/cmake.py index 04fbaa84885..440a1d0c288 100644 --- a/conans/client/build/cmake.py +++ b/conans/client/build/cmake.py @@ -111,9 +111,13 @@ def _generator(self): return "Unix Makefiles" def _toolset(self, toolset=None): - if "CONAN_CMAKE_TOOLSET" in os.environ: - return os.environ["CONAN_CMAKE_TOOLSET"] - return toolset + if toolset: + return toolset + elif self._settings.get_safe("compiler") == "Visual Studio": + subs_toolset = self._settings.get_safe("compiler.toolset") + if subs_toolset: + return subs_toolset + return None def _cmake_compiler_options(self, the_os, arch): cmake_definitions = OrderedDict() diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index fb42dc78f6d..1952dea7c5a 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -42,6 +42,7 @@ Visual Studio: runtime: [MD, MT, MTd, MDd] version: ["8", "9", "10", "11", "12", "14", "15"] + toolset: [None, v90, v100, v110, v110_xp, v120, v120_xp, v140, v140_xp, v140_clang_c2, LLVM-vs2014, LLVM-vs2014_xp, v141, v141_xp, v141_clang_c2] clang: version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0"] libcxx: [libstdc++, libstdc++11, libc++] diff --git a/conans/client/tools/win.py b/conans/client/tools/win.py index dea948d02a6..ffeb8159bd3 100644 --- a/conans/client/tools/win.py +++ b/conans/client/tools/win.py @@ -12,17 +12,18 @@ def msvc_build_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, - arch=None, parallel=True): + arch=None, parallel=True, force_vcvars=False, toolset=None): """ Do both: set the environment variables and call the .sln build """ - vcvars = vcvars_command(settings) - build = build_sln_command(settings, sln_path, targets, upgrade_project, build_type, arch, parallel) + vcvars = vcvars_command(settings, force=force_vcvars) + build = build_sln_command(settings, sln_path, targets, upgrade_project, build_type, arch, + parallel, toolset=toolset) command = "%s && %s" % (vcvars, build) return command def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, - arch=None, parallel=True): + arch=None, parallel=True, toolset=None): """ Use example: build_command = build_sln_command(self.settings, "myfile.sln", targets=["SDL2_image"]) @@ -50,6 +51,10 @@ def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, bu if targets: command += " /target:%s" % ";".join(targets) + + if toolset: + command += " /p:PlatformToolset=%s" % toolset + return command @@ -79,7 +84,7 @@ def vs_installation_path(version): return vs_installation_path._cached[version] -def vcvars_command(settings, arch=None, compiler_version=None): +def vcvars_command(settings, arch=None, compiler_version=None, force=False): arch_setting = arch or settings.get_safe("arch") compiler_version = compiler_version or settings.get_safe("compiler.version") if not compiler_version: @@ -91,9 +96,12 @@ def vcvars_command(settings, arch=None, compiler_version=None): command = "echo Conan:vcvars already set" existing_version = existing_version.split(".")[0] if existing_version != compiler_version: - raise ConanException("Error, Visual environment already set to %s\n" - "Current settings visual version: %s" - % (existing_version, compiler_version)) + message = "Visual environment already set to %s\n " \ + "Current settings visual version: %s" % (existing_version, compiler_version) + if not force: + raise ConanException("Error, %s" % message) + else: + _global_output.warn(message) else: env_var = "vs%s0comntools" % compiler_version diff --git a/conans/model/info.py b/conans/model/info.py index fecae68326d..1844e99bfe8 100644 --- a/conans/model/info.py +++ b/conans/model/info.py @@ -250,6 +250,8 @@ def create(settings, options, requires, indirect_requires): result.full_requires.extend(indirect_requires) result.recipe_hash = None result.env_values = EnvValues() + result.vs_toolset_compatible() + return result @staticmethod @@ -359,3 +361,28 @@ def header_only(self): self.settings.clear() self.options.clear() self.requires.unrelated_mode() + + def vs_toolset_compatible(self): + """Default behaviour, same package for toolset v140 with compiler=Visual Studio 15 than + using Visual Studio 14""" + + toolsets_versions = { + "v141": "15", + "v140": "14", + "v120": "12", + "v110": "11", + "v100": "10", + "v90": "9", + "v80": "8"} + + if self.full_settings.compiler == "Visual Studio": + toolset = str(self.full_settings.compiler.toolset) + if toolset in toolsets_versions: + self.settings.compiler.version = toolsets_versions[toolset] + self.settings.compiler.toolset = None + + def vs_toolset_incompatible(self): + """Will generate different packages for v140 and visual 15 than the visual 14""" + self.settings.compiler.version = self.full_settings.compiler.version + self.settings.compiler.toolset = self.full_settings.compiler.toolset + diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index f799b466703..5eab894d0c8 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -470,6 +470,7 @@ def set_toolset_test(self): settings.compiler = "Visual Studio" settings.compiler.version = "15" settings.arch = "x86" + settings.compiler.toolset = "v140" # Will be overwritten by parameter conan_file = ConanFileMock() conan_file.settings = settings @@ -477,9 +478,23 @@ def set_toolset_test(self): cmake = CMake(conan_file, toolset="v141") self.assertIn('-T "v141"', cmake.command_line) + # DEPRECATED VARIABLE, NOT MODIFY ANYMORE THE TOOLSET with tools.environment_append({"CONAN_CMAKE_TOOLSET": "v141"}): cmake = CMake(conan_file) - self.assertIn('-T "v141"', cmake.command_line) + self.assertNotIn('-T "v141"', cmake.command_line) + + settings = Settings.loads(default_settings_yml) + settings.os = "Windows" + settings.compiler = "Visual Studio" + settings.compiler.version = "15" + settings.arch = "x86" + settings.compiler.toolset = "v140" + + conan_file = ConanFileMock() + conan_file.settings = settings + + cmake = CMake(conan_file) + self.assertIn('-T "v140"', cmake.command_line) @staticmethod def scape(args): diff --git a/conans/test/integration/package_id_test.py b/conans/test/integration/package_id_test.py index db7597108fb..ba9d4e45bde 100644 --- a/conans/test/integration/package_id_test.py +++ b/conans/test/integration/package_id_test.py @@ -12,11 +12,12 @@ def setUp(self): self.client = TestClient() def _export(self, name, version, package_id_text=None, requires=None, - channel=None, default_option_value="off"): + channel=None, default_option_value="off", settings=None): conanfile = TestConanFile(name, version, requires=requires, options={"an_option": ["on", "off"]}, default_options=[("an_option", "%s" % default_option_value)], - package_id=package_id_text) + package_id=package_id_text, + settings=settings) self.client.save({"conanfile.py": str(conanfile)}, clean_first=True) self.client.run("export %s" % (channel or "lasote/stable")) @@ -168,3 +169,50 @@ def test_nameless_mode(self): self.client.save({"conanfile.txt": "[requires]\nHello2/2.3.8@lasote/stable"}, clean_first=True) # Not needed to rebuild Hello2, it doesn't matter its requires self.client.run("install .") + + def test_toolset_visual_compatibility(self): + # By default is the same to build with native visual or the toolchain + for package_id in [None, "self.info.vs_toolset_compatible()"]: + self._export("Hello", "1.2.0", package_id_text=package_id, + channel="user/testing", + settings='"compiler"') + self.client.run('install Hello/1.2.0@user/testing ' + ' -s compiler="Visual Studio" ' + ' -s compiler.version=14 --build') + + # Should have binary available + self.client.run('install Hello/1.2.0@user/testing' + ' -s compiler="Visual Studio" ' + ' -s compiler.version=15 -s compiler.toolset=v140') + + # Should NOT have binary available + error = self.client.run('install Hello/1.2.0@user/testing ' + '-s compiler="Visual Studio" ' + '-s compiler.version=15 -s compiler.toolset=v120', + ignore_error=True) + self.assertTrue(error) + self.assertIn("Missing prebuilt package for 'Hello/1.2.0@user/testing'", self.client.out) + + # Specify a toolset not involved with the visual version is ok, needed to build: + self.client.run('install Hello/1.2.0@user/testing' + ' -s compiler="Visual Studio" ' + ' -s compiler.version=15 -s compiler.toolset=v141_clang_c2 ' + '--build missing') + + def test_toolset_visual_incompatibility(self): + # By default is the same to build with native visual or the toolchain + self._export("Hello", "1.2.0", package_id_text="self.info.vs_toolset_incompatible()", + channel="user/testing", + settings='"compiler"', + ) + self.client.run('install Hello/1.2.0@user/testing ' + ' -s compiler="Visual Studio" ' + ' -s compiler.version=14 --build') + + # Should NOT have binary available + error = self.client.run('install Hello/1.2.0@user/testing' + ' -s compiler="Visual Studio" ' + ' -s compiler.version=15 -s compiler.toolset=v140', + ignore_error=True) + self.assertTrue(error) + self.assertIn("Missing prebuilt package for 'Hello/1.2.0@user/testing'", self.client.out) diff --git a/conans/test/util/build_sln_command_test.py b/conans/test/util/build_sln_command_test.py index 745f3d7926a..f81afa99b4c 100644 --- a/conans/test/util/build_sln_command_test.py +++ b/conans/test/util/build_sln_command_test.py @@ -55,3 +55,10 @@ def target_test(self): self.assertNotIn('devenv dummy.sln /upgrade', command) self.assertNotIn('/m:%s' % cpu_count(), command) self.assertIn('/target:teapot', command) + + def toolset_test(self): + command = build_sln_command(Settings({}), sln_path='dummy.sln', targets=None, + upgrade_project=False, build_type='Debug', arch='armv7', + parallel=False, toolset="v110") + self.assertEquals('msbuild dummy.sln /p:Configuration=Debug /p:Platform="ARM" ' + '/p:PlatformToolset=v110', command) diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py index 7b84a965d2b..32b1e86d4d9 100644 --- a/conans/test/util/tools_test.py +++ b/conans/test/util/tools_test.py @@ -366,8 +366,7 @@ def vcvars_echo_test(self): self.assertIn("VS140COMNTOOLS=", str(output)) def vcvars_constrained_test(self): - if platform.system() != "Windows": - return + text = """os: [Windows] compiler: Visual Studio: @@ -380,12 +379,17 @@ def vcvars_constrained_test(self): "compiler.version setting required for vcvars not defined"): tools.vcvars_command(settings) settings.compiler.version = "14" - cmd = tools.vcvars_command(settings) - self.assertIn("vcvarsall.bat", cmd) - with tools.environment_append({"VisualStudioVersion": "12"}): - with self.assertRaisesRegexp(ConanException, - "Error, Visual environment already set to 12"): - tools.vcvars_command(settings) + with tools.environment_append({"vs140comntools": "path/to/fake"}): + cmd = tools.vcvars_command(settings) + self.assertIn("vcvarsall.bat", cmd) + with tools.environment_append({"VisualStudioVersion": "12"}): + with self.assertRaisesRegexp(ConanException, + "Error, Visual environment already set to 12"): + tools.vcvars_command(settings) + + with tools.environment_append({"VisualStudioVersion": "12"}): + # Not raising + tools.vcvars_command(settings, force=True) def run_in_bash_test(self): if platform.system() != "Windows": From 7c209323f7032662e583dd443709f58d7b1997ff Mon Sep 17 00:00:00 2001 From: James Date: Thu, 16 Nov 2017 13:05:26 +0100 Subject: [PATCH 29/38] adding self.keep_imports feature (#2026) * adding self.keep_imports feature * moved keep_imports to class scope --- conans/client/importer.py | 2 ++ conans/client/installer.py | 13 +++++++------ conans/model/conan_file.py | 3 --- conans/test/functional/imports_tests.py | 24 ++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/conans/client/importer.py b/conans/client/importer.py index a77569c1f18..8ff82bfb5c5 100644 --- a/conans/client/importer.py +++ b/conans/client/importer.py @@ -62,6 +62,8 @@ def _report_save_manifest(copied_files, output, dest_folder, manifest_name): def run_imports(conanfile, dest_folder, output): + if not hasattr(conanfile, "imports"): + return [] file_importer = _FileImporter(conanfile, dest_folder) conanfile.copy = file_importer conanfile.imports_folder = dest_folder diff --git a/conans/client/installer.py b/conans/client/installer.py index 909492d4d76..9307d57f29f 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -197,12 +197,13 @@ def _build_package(self): export_folder = self._client_cache.export(self._conan_ref) self._conan_file.conanfile_directory = export_folder # Now remove all files that were imported with imports() - for f in copied_files: - try: - if f.startswith(self.build_folder): - os.remove(f) - except OSError: - self._out.warn("Unable to remove imported file from build: %s" % f) + if not getattr(self._conan_file, "keep_imports", False): + for f in copied_files: + try: + if f.startswith(self.build_folder): + os.remove(f) + except OSError: + self._out.warn("Unable to remove imported file from build: %s" % f) def _raise_package_not_found_error(conan_file, conan_ref, out): diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index c676d1875f3..bd5449c63f3 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -214,9 +214,6 @@ def configure(self): This is also the place for conditional requirements """ - def imports(self): - pass - def build(self): self.output.warn("This conanfile has no build step") diff --git a/conans/test/functional/imports_tests.py b/conans/test/functional/imports_tests.py index 5a93055e8bd..45930d13adc 100644 --- a/conans/test/functional/imports_tests.py +++ b/conans/test/functional/imports_tests.py @@ -2,6 +2,7 @@ from conans.test.utils.tools import TestClient from conans.util.files import load import os +from conans.model.ref import ConanFileReference, PackageReference conanfile = """from conans import ConanFile @@ -31,6 +32,29 @@ def _set_up(self): client.run("export lasote/testing") return client + def repackage_test(self): + client = self._set_up() + conanfile = """from conans import ConanFile +class TestConan(ConanFile): + requires='LibC/0.1@lasote/testing' + keep_imports = True + def imports(self): + self.copy("license*", dst="licenses", folder=True, ignore_case=True) + + def package(self): + self.copy("*") +""" + client.save({"conanfile.py": conanfile}, clean_first=True) + client.run("create Pkg/0.1@user/testing --build=missing") + self.assertIn("Pkg/0.1@user/testing package(): Copied 1 '.md' files: LICENSE.md", + client.out) + pkg_ref = PackageReference(ConanFileReference.loads("Pkg/0.1@user/testing"), + "e6f2dac07251ad9958120a7f7c324366fb3b6f2a") + pkg_folder = client.client_cache.package(pkg_ref) + self.assertTrue(os.path.exists(os.path.join(pkg_folder, "licenses/LibA/LICENSE.txt"))) + self.assertTrue(os.path.exists(os.path.join(pkg_folder, "licenses/LibB/LICENSE.md"))) + self.assertTrue(os.path.exists(os.path.join(pkg_folder, "licenses/LibC/license.txt"))) + def imports_folders_test(self): client = self._set_up() From fec37ccda95d1addc143ac06774b51edfcbecf31 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 17 Nov 2017 15:28:31 +0100 Subject: [PATCH 30/38] Feature/develop mode (#2039) * develop mode * develop mode * fixed broken tests due conan_info() * added install test and fixed bug --- conans/client/cmd/export.py | 2 - conans/client/conan_api.py | 2 +- conans/client/deps_builder.py | 8 +-- conans/client/loader.py | 4 ++ conans/client/manager.py | 65 ++++++++--------- conans/model/conan_file.py | 2 + conans/test/functional/develop_test.py | 85 +++++++++++++++++++++++ conans/test/model/transitive_reqs_test.py | 20 +++--- 8 files changed, 139 insertions(+), 49 deletions(-) create mode 100644 conans/test/functional/develop_test.py diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py index ad8f1f50f89..44d65ac5c8c 100644 --- a/conans/client/cmd/export.py +++ b/conans/client/cmd/export.py @@ -59,8 +59,6 @@ def _load_export_conanfile(conanfile_path, output, name, version): if not field_value: output.warn("Conanfile doesn't have '%s'.\n" "It is recommended to add it as attribute" % field) - if getattr(conanfile, "conan_info", None): - output.warn("conan_info() method is deprecated, use package_id() instead") try: # Exports is the only object field, we need to do this, because conan export needs it diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 79bd2200b47..bbe61910d6d 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -462,7 +462,7 @@ def install_reference(self, reference, settings=None, options=None, env=None, sc manifest_verify=manifest_verify, manifest_interactive=manifest_interactive, generators=generators, - cwd=cwd, deploy=True) + cwd=cwd, install_reference=True) @api_method def install(self, path="", settings=None, options=None, env=None, scope=None, diff --git a/conans/client/deps_builder.py b/conans/client/deps_builder.py index 3c0143ed1da..da3934ad24a 100644 --- a/conans/client/deps_builder.py +++ b/conans/client/deps_builder.py @@ -124,12 +124,8 @@ def propagate_info(self): indirect_reqs) # Once we are done, call package_id() to narrow and change possible values - if hasattr(conanfile, "conan_info"): - # Deprecated in 0.19 - conanfile.conan_info() - else: - with conanfile_exception_formatter(str(conanfile), "package_id"): - conanfile.package_id() + with conanfile_exception_formatter(str(conanfile), "package_id"): + conanfile.package_id() return ordered def direct_requires(self): diff --git a/conans/client/loader.py b/conans/client/loader.py index 248c00186eb..fcc2364f841 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -30,6 +30,7 @@ def __init__(self, runner, settings, profile): self._package_settings = profile.package_settings_values self._env_values = profile.env_values + self.dev_reference = None def load_conan(self, conanfile_path, output, consumer=False, reference=None): """ loads a ConanFile object from the given file @@ -68,6 +69,9 @@ def load_conan(self, conanfile_path, output, consumer=False, reference=None): result.scope = self._scopes.package_scope(result.name) result.in_local_cache = True + if consumer or (self.dev_reference and self.dev_reference == reference): + result.develop = True + return result except Exception as e: # re-raise with file name raise ConanException("%s: %s" % (conanfile_path, str(e))) diff --git a/conans/client/manager.py b/conans/client/manager.py index d620b5cfbca..9261f419440 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -110,14 +110,14 @@ def __init__(self, client_cache, user_io, runner, remote_manager, search_manager self._search_manager = search_manager self._settings_preprocessor = settings_preprocessor - def load_consumer_conanfile(self, conanfile_path, info_folder, output, reference=None, - deps_info_required=False): - + def _load_consumer_conanfile(self, conanfile_path, info_folder, output, + deps_info_required=False): + """loads a conanfile for local flow: source, imports, package, build + """ profile = read_conaninfo_profile(info_folder) or self._client_cache.default_profile loader = self.get_loader(profile) if conanfile_path.endswith(".py"): - consumer = not reference - conanfile = loader.load_conan(conanfile_path, output, consumer, reference) + conanfile = loader.load_conan(conanfile_path, output, consumer=True) else: conanfile = loader.load_conan_txt(conanfile_path, output) if deps_info_required is not None: @@ -125,6 +125,25 @@ def load_consumer_conanfile(self, conanfile_path, info_folder, output, reference return conanfile + def _load_install_conanfile(self, loader, reference_or_path, conanfile_filename, cwd=None): + """loads a conanfile for installation: install, info + cwd only used for virtuals, to pass it the current directory and make available the + conanfile.conanfile_directory (smell)""" + if isinstance(reference_or_path, ConanFileReference): + conanfile = loader.load_virtual([reference_or_path], cwd) + else: + output = ScopedOutput("PROJECT", self._user_io.out) + try: + if conanfile_filename and conanfile_filename.endswith(".txt"): + raise NotFoundException("") + conan_file_path = os.path.join(reference_or_path, conanfile_filename or CONANFILE) + conanfile = loader.load_conan(conan_file_path, output, consumer=True) + except NotFoundException: # Load conanfile.txt + conan_path = os.path.join(reference_or_path, conanfile_filename or CONANFILE_TXT) + conanfile = loader.load_conan_txt(conan_path, output) + + return conanfile + def get_loader(self, profile): self._client_cache.settings.values = profile.settings_values # Settings preprocessor @@ -204,24 +223,6 @@ def download(self, reference, package_ids, remote=None): else: remote_proxy.download_packages(reference, list(packages_props.keys())) - def _get_conanfile_object(self, loader, reference_or_path, conanfile_filename, cwd=None): - """cwd only used for virtuals, to pass it the current directory and make available the - conanfile.conanfile_directory (smell)""" - if isinstance(reference_or_path, ConanFileReference): - conanfile = loader.load_virtual([reference_or_path], cwd) - else: - output = ScopedOutput("PROJECT", self._user_io.out) - try: - if conanfile_filename and conanfile_filename.endswith(".txt"): - raise NotFoundException("") - conan_file_path = os.path.join(reference_or_path, conanfile_filename or CONANFILE) - conanfile = loader.load_conan(conan_file_path, output, consumer=True) - except NotFoundException: # Load conanfile.txt - conan_path = os.path.join(reference_or_path, conanfile_filename or CONANFILE_TXT) - conanfile = loader.load_conan_txt(conan_path, output) - - return conanfile - @staticmethod def _inject_require(conanfile, inject_require): """ test_package functionality requires injecting the tested package as requirement @@ -241,7 +242,7 @@ def _get_graph_builder(self, loader, update, remote_proxy): def _get_deps_graph(self, reference, profile, filename, remote_proxy): loader = self.get_loader(profile) - conanfile = self._get_conanfile_object(loader, reference, filename) + conanfile = self._load_install_conanfile(loader, reference, filename) graph_builder = self._get_graph_builder(loader, False, remote_proxy) deps_graph = graph_builder.load(conanfile) return deps_graph, graph_builder, conanfile @@ -298,7 +299,7 @@ def info_get_graph(self, reference, profile, remote=None, filename=None, check_u def install(self, reference, install_folder, profile, remote=None, build_modes=None, filename=None, update=False, manifest_folder=None, manifest_verify=False, manifest_interactive=False, - generators=None, no_imports=False, inject_require=None, cwd=None, deploy=False): + generators=None, no_imports=False, inject_require=None, cwd=None, install_reference=False): """ Fetch and build all dependencies for the given reference @param reference: ConanFileReference or path to user space conanfile @param install_folder: where the output files will be saved @@ -329,7 +330,9 @@ def install(self, reference, install_folder, profile, remote=None, update=update, manifest_manager=manifest_manager) loader = self.get_loader(profile) - conanfile = self._get_conanfile_object(loader, reference, filename, cwd=cwd) + if not install_reference and isinstance(reference, ConanFileReference): # is a create + loader.dev_reference = reference + conanfile = self._load_install_conanfile(loader, reference, filename, cwd=cwd) if inject_require: self._inject_require(conanfile, inject_require) graph_builder = self._get_graph_builder(loader, update, remote_proxy) @@ -388,7 +391,7 @@ def install(self, reference, install_folder, profile, remote=None, run_imports(conanfile, install_folder, output) call_system_requirements(conanfile, output) - if deploy: + if install_reference: # The conanfile loaded is really a virtual one. The one with the deploy is the first level one deploy_conanfile = deps_graph.inverse_levels()[1][0].conanfile if hasattr(deploy_conanfile, "deploy") and callable(deploy_conanfile.deploy): @@ -407,7 +410,7 @@ def source(self, conanfile_path, source_folder, info_folder): """ output = ScopedOutput("PROJECT", self._user_io.out) # only infos if exist - conanfile = self.load_consumer_conanfile(conanfile_path, info_folder, output) + conanfile = self._load_consumer_conanfile(conanfile_path, info_folder, output) config_source_local(source_folder, conanfile, output) def imports_undo(self, current_path): @@ -422,7 +425,7 @@ def imports(self, conan_file_path, dest_folder, info_folder): """ output = ScopedOutput("PROJECT", self._user_io.out) - conanfile = self.load_consumer_conanfile(conan_file_path, info_folder, + conanfile = self._load_consumer_conanfile(conan_file_path, info_folder, output, deps_info_required=True) run_imports(conanfile, dest_folder, output) @@ -434,7 +437,7 @@ def local_package(self, package_folder, recipe_folder, build_folder, source_fold "--build_folder and package folder can't be the same") output = ScopedOutput("PROJECT", self._user_io.out) conan_file_path = os.path.join(recipe_folder, CONANFILE) - conanfile = self.load_consumer_conanfile(conan_file_path, install_folder, output, + conanfile = self._load_consumer_conanfile(conan_file_path, install_folder, output, deps_info_required=True) packager.create_package(conanfile, source_folder, build_folder, package_folder, install_folder, output, local=True, copy_info=True) @@ -451,7 +454,7 @@ def build(self, conanfile_path, source_folder, build_folder, package_folder, ins # Append env_vars to execution environment and clear when block code ends output = ScopedOutput(("%s test package" % test) if test else "Project", self._user_io.out) - conan_file = self.load_consumer_conanfile(conanfile_path, install_folder, output, + conan_file = self._load_consumer_conanfile(conanfile_path, install_folder, output, deps_info_required=True) except NotFoundException: # TODO: Auto generate conanfile from requirements file diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index bd5449c63f3..c7d03f3e5f9 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -125,6 +125,8 @@ def __init__(self, output, runner, settings, conanfile_directory, user=None, cha self.conanfile_directory = conanfile_directory self._scope = None + self.develop = False + # user specified env variables self._env_values = EnvValues() # Updated at runtime, user specified -e self._user = user diff --git a/conans/test/functional/develop_test.py b/conans/test/functional/develop_test.py new file mode 100644 index 00000000000..755916f73fb --- /dev/null +++ b/conans/test/functional/develop_test.py @@ -0,0 +1,85 @@ +import unittest +from conans.test.utils.tools import TestClient + + +conanfile = """from conans import ConanFile +class Pkg(ConanFile): + def requirements(self): + if self.develop: + self.output.info("Develop requirements!") + def source(self): + if self.develop: + self.output.info("Develop source!") + def build(self): + if self.develop: + self.output.info("Develop build!") + def package(self): + if self.develop: + self.output.info("Develop package!") + def package_info(self): + if self.develop: + self.output.info("Develop package_info!") + def package_id(self): + if self.develop: + self.output.info("Develop package_id!") +""" + + +class DevelopTest(unittest.TestCase): + + def develop_test(self): + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.run("create Pkg/0.1@user/testing") + self.assertIn("Develop requirements!", client.out) + self.assertIn("Develop source!", client.out) + self.assertIn("Develop build!", client.out) + self.assertIn("Develop package!", client.out) + self.assertIn("Develop package_info!", client.out) + self.assertIn("Develop package_id!", client.out) + + client.run("install Pkg/0.1@user/testing --build") + self.assertNotIn("Develop", client.out) + + consumer = """from conans import ConanFile +class Pkg(ConanFile): + requires = "Pkg/0.1@user/testing" +""" + client.save({"conanfile.py": consumer}) + client.run("create Other/1.0@user/testing") + self.assertNotIn("Develop", client.out) + + def local_commands_test(self): + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.run("install .") + self.assertIn("Develop requirements!", client.out) + self.assertNotIn("Develop source!", client.out) + self.assertNotIn("Develop build!", client.out) + self.assertNotIn("Develop package!", client.out) + self.assertNotIn("Develop package_info!", client.out) + self.assertIn("Develop package_id!", client.out) + + client.run("source .") + self.assertNotIn("Develop requirements!", client.out) + self.assertIn("Develop source!", client.out) + self.assertNotIn("Develop build!", client.out) + self.assertNotIn("Develop package!", client.out) + self.assertNotIn("Develop package_info!", client.out) + self.assertNotIn("Develop package_id!", client.out) + + client.run("build .") + self.assertNotIn("Develop requirements!", client.out) + self.assertNotIn("Develop source!", client.out) + self.assertIn("Develop build!", client.out) + self.assertNotIn("Develop package!", client.out) + self.assertNotIn("Develop package_info!", client.out) + self.assertNotIn("Develop package_id!", client.out) + + client.run("package .") + self.assertNotIn("Develop requirements!", client.out) + self.assertNotIn("Develop source!", client.out) + self.assertNotIn("Develop build!", client.out) + self.assertIn("Develop package!", client.out) + self.assertNotIn("Develop package_info!", client.out) + self.assertNotIn("Develop package_id!", client.out) diff --git a/conans/test/model/transitive_reqs_test.py b/conans/test/model/transitive_reqs_test.py index 301201d1950..0618d0b69ba 100644 --- a/conans/test/model/transitive_reqs_test.py +++ b/conans/test/model/transitive_reqs_test.py @@ -41,6 +41,7 @@ def get_recipe(self, conan_ref): conan_path = os.path.join(self.folder, "/".join(conan_ref), CONANFILE) return conan_path + say_content = """ from conans import ConanFile @@ -329,7 +330,7 @@ class ChatConan(ConanFile): version = "2.3" requires = "Hello/1.2@user/testing" - def conan_info(self): + def package_id(self): hello_require = self.info.requires["Hello"] hello_require.version = hello_require.full_version.minor() say_require = self.info.requires["Say"] @@ -377,7 +378,7 @@ class ChatConan(ConanFile): version = "2.3" requires = "Hello/1.2@user/testing" - def conan_info(self): + def package_id(self): self.info.requires["Hello"].full_package_mode() self.info.requires["Say"].semver_mode() """ @@ -1391,7 +1392,7 @@ class HelloConan(ConanFile): version = "1.2" requires = "Say/0.1@user/testing" - def conan_info(self): + def package_id(self): self.info.requires.clear() """ @@ -1414,7 +1415,7 @@ class HelloConan(ConanFile): version = "1.2" requires = "Say/0.1@user/testing" - def conan_info(self): + def package_id(self): self.info.requires.remove("Say") """ @@ -1437,7 +1438,7 @@ class ChatConan(ConanFile): version = "1.2" requires = "Hello/1.2@user/testing", "Bye/0.2@user/testing" - def conan_info(self): + def package_id(self): self.info.requires.remove("Bye", "Hello") """ @@ -1476,7 +1477,7 @@ class HelloConan(ConanFile): options = {"shared": [True, False]} default_options = "shared=True" - def conan_info(self): + def package_id(self): if self.options.shared: self.info.options["Say"] = self.info.full_options["Say"] """ @@ -1491,7 +1492,7 @@ class ChatConan(ConanFile): options = {"shared": [True, False]} default_options = "shared=True" - def conan_info(self): + def package_id(self): if self.options.shared: self.info.options["Hello"] = self.info.full_options["Hello"] self.info.options["Say"].shared = self.info.full_options["Say"].shared @@ -1729,7 +1730,7 @@ class SayConan(ConanFile): settings = "os" options = {"myoption": [1, 2, 3]} - def conan_info(self): + def package_id(self): self.info.settings.os = "Win" self.info.options.myoption = "1,2,3" """ @@ -1867,7 +1868,8 @@ def check(conanfile, options, settings): with self.assertRaises(ConanException) as cm: self.root(content, options="arch_independent=True", settings="os=Linux") self.assertIn(bad_value_msg("settings.os", "Linux", - ['Android', 'Arduino', 'FreeBSD', 'Macos', 'SunOS', "Windows", "iOS", "tvOS", "watchOS"]), + ['Android', 'Arduino', 'FreeBSD', 'Macos', + 'SunOS', "Windows", "iOS", "tvOS", "watchOS"]), str(cm.exception)) def test_config_remove2(self): From d38985cad8c3b807850b54053b47451303f2aec2 Mon Sep 17 00:00:00 2001 From: Santiago Alessandri Date: Sun, 19 Nov 2017 14:31:30 -0800 Subject: [PATCH 31/38] Set default output directory in RelWithDebInfo builds (#2041) * Set default output directory in RelWithDebInfo builds Solves issue #2032 * Fix default output directory for MinSizeRel build type --- conans/client/generators/cmake_common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conans/client/generators/cmake_common.py b/conans/client/generators/cmake_common.py index e4d1711c06b..1c4e86de523 100644 --- a/conans/client/generators/cmake_common.py +++ b/conans/client/generators/cmake_common.py @@ -252,10 +252,14 @@ def generate_targets_section(dependencies): macro(conan_output_dirs_setup) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) endmacro() From dbb83f3e38fd2b2cbd2882cf3607c526c6b5a1f6 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 20 Nov 2017 18:00:09 +0100 Subject: [PATCH 32/38] Feature/fix profiles paths (#2045) * fixed profile paths * also Windows local path * fixed profile paths * fixed broken test --- conans/client/conan_api.py | 3 +- conans/client/profile_loader.py | 39 ++++++++++--------- conans/test/command/install_test.py | 29 +++++++++++++- conans/test/functional/profile_loader_test.py | 28 ++++++------- conans/test/integration/profile_test.py | 4 +- 5 files changed, 64 insertions(+), 39 deletions(-) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index bbe61910d6d..bfa080c35a8 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -775,7 +775,8 @@ def profile_list(self): @api_method def create_profile(self, profile_name, detect=False): - profile_path = get_profile_path(profile_name, self._client_cache.profiles_path, os.getcwd()) + profile_path = get_profile_path(profile_name, self._client_cache.profiles_path, os.getcwd(), + exists=False) if os.path.exists(profile_path): raise ConanException("Profile already exists") diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index fbbd9fa924f..e2501915b83 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -83,16 +83,27 @@ def read_conaninfo_profile(current_path): return profile -def get_profile_path(profile_name, default_folder, cwd): +def get_profile_path(profile_name, default_folder, cwd, exists=True): + def valid_path(profile_path): + if exists and not os.path.isfile(profile_path): + raise ConanException("Profile not found: %s" % profile_path) + return profile_path + if os.path.isabs(profile_name): - profile_path = profile_name - elif cwd and (os.path.exists(os.path.join(cwd, profile_name)) or profile_name.startswith(".")): - # relative path name + return valid_path(profile_name) + + if profile_name[:2] in ("./", ".\\"): # local profile_path = os.path.abspath(os.path.join(cwd, profile_name)) - else: - if not os.path.exists(default_folder): - mkdir(default_folder) - profile_path = os.path.join(default_folder, profile_name) + return valid_path(profile_path) + + if not os.path.exists(default_folder): + mkdir(default_folder) + profile_path = os.path.join(default_folder, profile_name) + if exists: + if not os.path.isfile(profile_path): + profile_path = os.path.abspath(os.path.join(cwd, profile_name)) + if not os.path.isfile(profile_path): + raise ConanException("Profile not found: %s" % profile_name) return profile_path @@ -105,17 +116,7 @@ def read_profile(profile_name, cwd, default_folder): return None, None profile_path = get_profile_path(profile_name, default_folder, cwd) - try: - text = load(profile_path) - except IOError: - folder = os.path.dirname(profile_path) - if os.path.exists(folder): - profiles = [name for name in os.listdir(folder) if not os.path.isdir(name)] - else: - profiles = [] - current_profiles = ", ".join(profiles) or "[]" - raise ConanException("Specified profile '%s' doesn't exist.\nExisting profiles: " - "%s" % (profile_name, current_profiles)) + text = load(profile_path) try: return _load_profile(text, profile_path, default_folder) diff --git a/conans/test/command/install_test.py b/conans/test/command/install_test.py index cae36aa64e9..88b3702cac0 100644 --- a/conans/test/command/install_test.py +++ b/conans/test/command/install_test.py @@ -9,7 +9,7 @@ from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.paths import CONANFILE_TXT from conans.client.conf.detect import detected_os -from conans.util.files import load +from conans.util.files import load, mkdir class InstallTest(unittest.TestCase): @@ -299,3 +299,30 @@ class TestConan(ConanFile): client.save({}, clean_first=True) client.run("install Hello/0.1@conan/stable") self.assertFalse(os.path.exists(os.path.join(client.current_folder, "conanbuildinfo.txt"))) + + def install_with_profile_test(self): + # Test for https://github.com/conan-io/conan/pull/2043 + conanfile = """from conans import ConanFile +class TestConan(ConanFile): + settings = "os" + def requirements(self): + self.output.info("PKGOS=%s" % self.settings.os) +""" + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.run("profile new myprofile") + client.run("profile update settings.os=Linux myprofile") + client.run("install . -pr=myprofile --build") + self.assertIn("PKGOS=Linux", client.out) + mkdir(os.path.join(client.current_folder, "myprofile")) + client.run("install . -pr=myprofile") + client.run("profile new myotherprofile") + client.run("profile update settings.os=FreeBSD myotherprofile") + client.run("install . -pr=myotherprofile") + self.assertIn("PKGOS=FreeBSD", client.out) + client.save({"myotherprofile": "Some garbage without sense [garbage]"}) + client.run("install . -pr=myotherprofile") + self.assertIn("PKGOS=FreeBSD", client.out) + error = client.run("install . -pr=./myotherprofile", ignore_error=True) + self.assertTrue(error) + self.assertIn("Error parsing the profile", client.out) diff --git a/conans/test/functional/profile_loader_test.py b/conans/test/functional/profile_loader_test.py index c8d17e70a21..4e7a3980df5 100644 --- a/conans/test/functional/profile_loader_test.py +++ b/conans/test/functional/profile_loader_test.py @@ -177,12 +177,11 @@ def profile_loads_win_test(self): self.assertIn("QTPATH2=C:/QtCommercial2/5.8/msvc2015_64/bin", new_profile.dumps()) def profile_load_dump_test(self): - # Empty profile tmp = temp_folder() profile = Profile() - dump = profile.dumps() - new_profile, _ = self._get_profile(tmp, "") + dumps = profile.dumps() + new_profile, _ = self._get_profile(tmp, dumps) self.assertEquals(new_profile.settings, profile.settings) # Settings @@ -299,8 +298,8 @@ def install_with_missing_profile_test(self, path): self.client.save({CONANFILE: conanfile_scope_env}) error = self.client.run('install -pr "%sscopes_env"' % path, ignore_error=True) self.assertTrue(error) - self.assertIn("ERROR: Specified profile '%sscopes_env' doesn't exist" % path, - self.client.user_io.out) + self.assertIn("ERROR: Profile not found: ", self.client.out) + self.assertIn("scopes_env", self.client.out) def install_profile_env_test(self): files = cpp_hello_conan_files("Hello0", "0.1", build=False) @@ -564,7 +563,7 @@ def profile_vars_test(self): [env] MYVAR=$MY_MAGIC_VAR what they seem. ''' - profile, vars = self._get_profile(tmp, txt) + profile, _ = self._get_profile(tmp, txt) self.assertEquals("The owls are not what they seem.", profile.env_values.data[None]["MYVAR"]) # Order in replacement, variable names (simplification of preprocessor) @@ -575,7 +574,7 @@ def profile_vars_test(self): [env] MYVAR=$P2 ''' - profile, vars = self._get_profile(tmp, txt) + profile, _ = self._get_profile(tmp, txt) self.assertEquals("Diane, the coffee at the Great Northern2", profile.env_values.data[None]["MYVAR"]) # Variables without spaces @@ -588,8 +587,8 @@ def profile_vars_test(self): self._get_profile(tmp, txt) def test_profiles_includes(self): - tmp = temp_folder() + def save_profile(txt, name): abs_profile_path = os.path.join(tmp, name) save(abs_profile_path, txt) @@ -610,11 +609,7 @@ def save_profile(txt, name): profile1 = """ # Include in subdir, curdir MYVAR=1 -include(profile0.txt) - - - - +include(./profile0.txt) [settings] os=Windows @@ -631,7 +626,7 @@ def save_profile(txt, name): profile2 = """ # Include in subdir -include(subdir/profile1.txt) +include(./subdir/profile1.txt) [settings] os=$MYVAR """ @@ -664,9 +659,10 @@ def save_profile(txt, name): save_profile(profile4, "profile4.txt") - profile, vars = read_profile("./profile4.txt", tmp, None) + profile, variables = read_profile("./profile4.txt", tmp, None) - self.assertEquals(vars, {"MYVAR": "1", "OTHERVAR": "34", "PROFILE_DIR": tmp , "ROOTVAR": "0"}) + self.assertEquals(variables, {"MYVAR": "1", "OTHERVAR": "34", "PROFILE_DIR": + tmp, "ROOTVAR": "0"}) self.assertEquals("FromProfile3And34", profile.env_values.data[None]["MYVAR"]) self.assertEquals("1", profile.env_values.data["package1"]["ENVY"]) self.assertEquals(profile.settings, {"os": "1"}) diff --git a/conans/test/integration/profile_test.py b/conans/test/integration/profile_test.py index ef977021b80..d22ca933e9d 100644 --- a/conans/test/integration/profile_test.py +++ b/conans/test/integration/profile_test.py @@ -135,8 +135,8 @@ def install_with_missing_profile_test(self, path): self.client.save({CONANFILE: conanfile_scope_env}) error = self.client.run('install -pr "%sscopes_env"' % path, ignore_error=True) self.assertTrue(error) - self.assertIn("ERROR: Specified profile '%sscopes_env' doesn't exist" % path, - self.client.user_io.out) + self.assertIn("ERROR: Profile not found:", self.client.out) + self.assertIn("scopes_env", self.client.out) def install_profile_env_test(self): files = cpp_hello_conan_files("Hello0", "0.1", build=False) From 9a1ed92c9b28489abc4c80ffab82c4a10bdd0664 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Tue, 21 Nov 2017 00:01:40 +0700 Subject: [PATCH 33/38] - auto-detect MSYS2 (for system package tool) (#1964) --- conans/client/tools/oss.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conans/client/tools/oss.py b/conans/client/tools/oss.py index 4be610b7506..2d3998b89e7 100644 --- a/conans/client/tools/oss.py +++ b/conans/client/tools/oss.py @@ -6,6 +6,7 @@ import os from conans.model.version import Version from conans.util.log import logger +from conans.client.tools import which _global_output = None @@ -102,7 +103,12 @@ def with_yum(self): @property def with_pacman(self): - return self.is_linux and self.linux_distro == "arch" + if self.is_linux: + return self.linux_distro == "arch" + elif self.is_windows and which('uname.exe'): + uname = subprocess.check_output(['uname.exe', '-s']).decode() + return uname.startswith('MSYS_NT') and which('pacman.exe') + return False @staticmethod def get_win_os_version(): From 82a6ab0cf612dda9414ccc70c8fef1d4ccc87012 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Tue, 21 Nov 2017 00:04:55 +0700 Subject: [PATCH 34/38] - define CMAKE_LIBRARY_OUTPUT_DIRECTORY as well (#2016) * - define CMAKE_LIBRARY_OUTPUT_DIRECTORY as well * - OSX needs DYLD_LIBRARY_PATH --- conans/client/generators/cmake_common.py | 6 ++++++ conans/test/integration/basic_build_test.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/conans/client/generators/cmake_common.py b/conans/client/generators/cmake_common.py index 1c4e86de523..0c8a43f079c 100644 --- a/conans/client/generators/cmake_common.py +++ b/conans/client/generators/cmake_common.py @@ -261,6 +261,12 @@ def generate_targets_section(dependencies): set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) + + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELWITHDEBINFO ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_MINSIZEREL ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) endmacro() macro(conan_split_version VERSION_STRING MAJOR MINOR) diff --git a/conans/test/integration/basic_build_test.py b/conans/test/integration/basic_build_test.py index 9e22d4f3c7c..7ba79f53554 100644 --- a/conans/test/integration/basic_build_test.py +++ b/conans/test/integration/basic_build_test.py @@ -24,6 +24,8 @@ def _build(self, cmd, static, pure_c, use_cmake, lang): client.run('build .') ld_path = ("LD_LIBRARY_PATH=`pwd`" if not static and not platform.system() == "Windows" else "") + if platform.system() == "Darwin": + ld_path += ' DYLD_LIBRARY_PATH="%s"' % os.path.join(client.current_folder, 'lib') command = os.sep.join([".", "bin", "say_hello"]) client.runner("%s %s" % (ld_path, command), cwd=client.current_folder) msg = "Hello" if lang == 0 else "Hola" From b0c7bd57b6076baa53bff307ed039fa1912da5e7 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Tue, 21 Nov 2017 04:19:20 +0700 Subject: [PATCH 35/38] support WindowsStore OS (#1987) * - support for arm and arm64 architectures in vcvars_command * - support for ARM64 in build_sln_command * - add WindowsStore as supported OS * - use /MP for WindowsStore as well * - handle WindowsStore in vcvars_command * - test CMake settings for WindowsStore * - fix failing tests * - vcvars_command assumes x64 by default to match previous implementation * - raise an human-readable exception if Windows 10 SDK wasn't found --- conans/client/build/cmake.py | 2 +- conans/client/conf/__init__.py | 2 + conans/client/tools/win.py | 74 +++++++++++++++--- conans/test/functional/cmake_test.py | 21 ++++- .../test/functional/compile_helpers_test.py | 2 +- conans/test/model/other_settings_test.py | 3 +- conans/test/model/transitive_reqs_test.py | 3 +- conans/test/util/build_sln_command_test.py | 4 +- conans/test/util/vcvars_arch_test.py | 73 ++++++++++++++++++ conans/test/util/vcvars_store_test.py | 77 +++++++++++++++++++ 10 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 conans/test/util/vcvars_arch_test.py create mode 100644 conans/test/util/vcvars_store_test.py diff --git a/conans/client/build/cmake.py b/conans/client/build/cmake.py index 440a1d0c288..836dfa0e88b 100644 --- a/conans/client/build/cmake.py +++ b/conans/client/build/cmake.py @@ -273,7 +273,7 @@ def _get_cmake_definitions(self): except AttributeError: pass - if self._os == "Windows" and self._compiler == "Visual Studio": + if str(self._os) in ["Windows", "WindowsStore"] and self._compiler == "Visual Studio": if self.parallel: cpus = tools.cpu_count() ret["CONAN_CXX_FLAGS"] = "/MP%s" % cpus diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 1952dea7c5a..8aab3612053 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -14,6 +14,8 @@ default_settings_yml = """ os: Windows: + WindowsStore: + version: ["8.1", "10.0"] Linux: Macos: Android: diff --git a/conans/client/tools/win.py b/conans/client/tools/win.py index ffeb8159bd3..5df96bbed13 100644 --- a/conans/client/tools/win.py +++ b/conans/client/tools/win.py @@ -5,7 +5,7 @@ from conans.client.tools.env import environment_append from conans.client.tools.files import unix_path -from conans.client.tools.oss import cpu_count +from conans.client.tools.oss import cpu_count, detected_architecture from conans.errors import ConanException _global_output = None @@ -40,11 +40,12 @@ def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, bu raise ConanException("Cannot build_sln_command, arch not defined") command += "msbuild %s /p:Configuration=%s" % (sln_path, build_type) arch = str(arch) - if arch in ["x86_64", "x86"]: - command += ' /p:Platform=' - command += '"x64"' if arch == "x86_64" else '"x86"' - elif "ARM" in arch.upper(): - command += ' /p:Platform="ARM"' + msvc_arch = {'x86': 'x86', + 'x86_64': 'x64', + 'armv7': 'ARM', + 'armv8': 'ARM64'}.get(arch) + if msvc_arch: + command += ' /p:Platform="%s"' % msvc_arch if parallel: command += ' /m:%s' % cpu_count() @@ -84,13 +85,55 @@ def vs_installation_path(version): return vs_installation_path._cached[version] +def find_windows_10_sdk(): + """finds valid Windows 10 SDK version which can be passed to vcvarsall.bat (vcvars_command)""" + # uses the same method as VCVarsQueryRegistry.bat + from six.moves import winreg # @UnresolvedImport + hives = [ + (winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Wow6432Node'), + (winreg.HKEY_CURRENT_USER, r'SOFTWARE\Wow6432Node'), + (winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE'), + (winreg.HKEY_CURRENT_USER, r'SOFTWARE') + ] + for key, subkey in hives: + try: + hkey = winreg.OpenKey(key, r'%s\Microsoft\Microsoft SDKs\Windows\v10.0' % subkey) + installation_folder, _ = winreg.QueryValueEx(hkey, 'InstallationFolder') + if os.path.isdir(installation_folder): + include_dir = os.path.join(installation_folder, 'include') + for sdk_version in os.listdir(include_dir): + if os.path.isdir(os.path.join(include_dir, sdk_version)) and sdk_version.startswith('10.'): + windows_h = os.path.join(include_dir, sdk_version, 'um', 'Windows.h') + if os.path.isfile(windows_h): + return sdk_version + except EnvironmentError: + pass + finally: + winreg.CloseKey(hkey) + return None + + def vcvars_command(settings, arch=None, compiler_version=None, force=False): arch_setting = arch or settings.get_safe("arch") compiler_version = compiler_version or settings.get_safe("compiler.version") + os_setting = settings.get_safe("os") if not compiler_version: raise ConanException("compiler.version setting required for vcvars not defined") - param = "x86" if arch_setting == "x86" else "amd64" + # https://msdn.microsoft.com/en-us/library/f2ccy3wt.aspx + arch_setting = arch_setting or 'x86_64' + if detected_architecture() == 'x86_64': + vcvars_arch = {'x86': 'x86', + 'x86_64': 'amd64', + 'armv7': 'amd64_arm', + 'armv8': 'amd64_arm64'}.get(arch_setting) + elif detected_architecture() == 'x86': + vcvars_arch = {'x86': 'x86', + 'x86_64': 'x86_amd64', + 'armv7': 'x86_arm', + 'armv8': 'x86_arm64'}.get(arch_setting) + if not vcvars_arch: + raise ConanException('unsupported architecture %s' % arch_setting) existing_version = os.environ.get("VisualStudioVersion") if existing_version: command = "echo Conan:vcvars already set" @@ -119,7 +162,7 @@ def vcvars_command(settings, arch=None, compiler_version=None, force=False): 'please check that you have set it correctly' % (env_var, vs_path)) vcvars_path = os.path.join(vs_path, "../../VC/Auxiliary/Build/vcvarsall.bat") command = ('set "VSCMD_START_DIR=%%CD%%" && ' - 'call "%s" %s' % (vcvars_path, param)) + 'call "%s" %s' % (vcvars_path, vcvars_arch)) else: try: vs_path = os.environ[env_var] @@ -129,8 +172,19 @@ def vcvars_command(settings, arch=None, compiler_version=None, force=False): _global_output.warn('VS variable %s points to the non-existing path "%s",' 'please check that you have set it correctly' % (env_var, vs_path)) vcvars_path = os.path.join(vs_path, "../../VC/vcvarsall.bat") - command = ('call "%s" %s' % (vcvars_path, param)) - + command = ('call "%s" %s' % (vcvars_path, vcvars_arch)) + + if os_setting == 'WindowsStore': + os_version_setting = settings.get_safe("os.version") + if os_version_setting == '8.1': + command += ' store 8.1' + elif os_version_setting == '10.0': + windows_10_sdk = find_windows_10_sdk() + if not windows_10_sdk: + raise ConanException("cross-compiling for WindowsStore 10 (UWP), but Windows 10 SDK wasn't found") + command += ' store ' + windows_10_sdk + else: + raise ConanException('unsupported Windows Store version %s' % os_version_setting) return command diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 5eab894d0c8..10f965df410 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -79,9 +79,11 @@ def loads_default_test(self): def check(text, build_config, generator=None): os = str(settings.os) + os_ver = str(settings.os.version) if settings.get_safe('os.version') else None for cmake_system_name in (True, False): - cross = ("-DCMAKE_SYSTEM_NAME=\"%s\" -DCMAKE_SYSROOT=\"/path/to/sysroot\" " - % {"Macos": "Darwin"}.get(os, os) + cross_ver = ("-DCMAKE_SYSTEM_VERSION=\"%s\" " % os_ver) if os_ver else "" + cross = ("-DCMAKE_SYSTEM_NAME=\"%s\" %s-DCMAKE_SYSROOT=\"/path/to/sysroot\" " + % ({"Macos": "Darwin"}.get(os, os), cross_ver) if (platform.system() != os and cmake_system_name) else "") cmake = CMake(conan_file, generator=generator, cmake_system_name=cmake_system_name) new_text = text.replace("-DCONAN_EXPORTED", "%s-DCONAN_EXPORTED" % cross) @@ -180,6 +182,21 @@ def check(text, build_config, generator=None): '-DCONAN_SHARED_LINKER_FLAGS="-m64" -DCONAN_C_FLAGS="-m64" -Wno-dev', "") + settings.compiler = "Visual Studio" + settings.compiler.version = "12" + settings.os = "WindowsStore" + settings.os.version = "8.1" + settings.build_type = "Debug" + check('-G "Visual Studio 12 2013" -DCONAN_EXPORTED="1" ' + '-DCONAN_COMPILER="Visual Studio" -DCONAN_COMPILER_VERSION="12" -Wno-dev', + "--config Debug") + + settings.os.version = "10.0" + check('-G "Visual Studio 12 2013" -DCONAN_EXPORTED="1" ' + '-DCONAN_COMPILER="Visual Studio" -DCONAN_COMPILER_VERSION="12" -Wno-dev', + "--config Debug") + + def deleted_os_test(self): partial_settings = """ os: [Linux] diff --git a/conans/test/functional/compile_helpers_test.py b/conans/test/functional/compile_helpers_test.py index 9809fd378a9..7acaadfe65f 100644 --- a/conans/test/functional/compile_helpers_test.py +++ b/conans/test/functional/compile_helpers_test.py @@ -53,7 +53,7 @@ def libcxx(self): @property def os(self): - return self._os + return MockSetting(self._os) @property def arch(self): diff --git a/conans/test/model/other_settings_test.py b/conans/test/model/other_settings_test.py index df600d34b04..c5e0e7c1465 100644 --- a/conans/test/model/other_settings_test.py +++ b/conans/test/model/other_settings_test.py @@ -148,7 +148,8 @@ class SayConan(ConanFile): self.client.save({CONANFILE: content}) self.client.run("install -s os=ChromeOS --build missing", ignore_error=True) self.assertIn(bad_value_msg("settings.os", "ChromeOS", - ['Android', 'Arduino', 'FreeBSD', 'Linux', 'Macos', 'SunOS', 'Windows', 'iOS', 'tvOS', 'watchOS']), + ['Android', 'Arduino', 'FreeBSD', 'Linux', 'Macos', 'SunOS', 'Windows', + 'WindowsStore', 'iOS', 'tvOS', 'watchOS']), str(self.client.user_io.out)) # Now add new settings to config and try again diff --git a/conans/test/model/transitive_reqs_test.py b/conans/test/model/transitive_reqs_test.py index 0618d0b69ba..831458b9ed6 100644 --- a/conans/test/model/transitive_reqs_test.py +++ b/conans/test/model/transitive_reqs_test.py @@ -1869,7 +1869,8 @@ def check(conanfile, options, settings): self.root(content, options="arch_independent=True", settings="os=Linux") self.assertIn(bad_value_msg("settings.os", "Linux", ['Android', 'Arduino', 'FreeBSD', 'Macos', - 'SunOS', "Windows", "iOS", "tvOS", "watchOS"]), + 'SunOS', 'Windows', 'WindowsStore', + 'iOS', 'tvOS', 'watchOS']), str(cm.exception)) def test_config_remove2(self): diff --git a/conans/test/util/build_sln_command_test.py b/conans/test/util/build_sln_command_test.py index f81afa99b4c..fec040b2154 100644 --- a/conans/test/util/build_sln_command_test.py +++ b/conans/test/util/build_sln_command_test.py @@ -49,9 +49,9 @@ def parallel_test(self): def target_test(self): command = build_sln_command(Settings({}), sln_path='dummy.sln', targets=['teapot'], upgrade_project=False, - build_type='Debug', arch='x86', parallel=False) + build_type='Debug', arch='armv8', parallel=False) self.assertIn('msbuild dummy.sln', command) - self.assertIn('/p:Platform="x86"', command) + self.assertIn('/p:Platform="ARM64"', command) self.assertNotIn('devenv dummy.sln /upgrade', command) self.assertNotIn('/m:%s' % cpu_count(), command) self.assertIn('/target:teapot', command) diff --git a/conans/test/util/vcvars_arch_test.py b/conans/test/util/vcvars_arch_test.py new file mode 100644 index 00000000000..34c03e0bb32 --- /dev/null +++ b/conans/test/util/vcvars_arch_test.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import platform +import unittest +from nose.plugins.attrib import attr +from conans.model.settings import Settings +from conans.client.conf import default_settings_yml +from conans.errors import ConanException +from conans import tools + + +@attr('visual_studio') +class VCVarsArchTest(unittest.TestCase): + def test_arch(self): + if platform.system() != "Windows": + return + settings = Settings.loads(default_settings_yml) + settings.compiler = 'Visual Studio' + settings.compiler.version = '14' + + settings.arch = 'x86' + command = tools.vcvars_command(settings) + self.assertIn('vcvarsall.bat', command) + self.assertIn('x86', command) + + settings.arch = 'x86_64' + command = tools.vcvars_command(settings) + self.assertIn('vcvarsall.bat', command) + self.assertIn('amd64', command) + + settings.arch = 'armv7' + command = tools.vcvars_command(settings) + self.assertIn('vcvarsall.bat', command) + self.assertNotIn('arm64', command) + self.assertIn('arm', command) + + settings.arch = 'armv8' + command = tools.vcvars_command(settings) + self.assertIn('vcvarsall.bat', command) + self.assertIn('arm64', command) + + settings.arch = 'mips' + with self.assertRaises(ConanException): + tools.vcvars_command(settings) + + def test_arch_override(self): + if platform.system() != "Windows": + return + settings = Settings.loads(default_settings_yml) + settings.compiler = 'Visual Studio' + settings.compiler.version = '14' + settings.arch = 'mips64' + + command = tools.vcvars_command(settings, arch='x86') + self.assertIn('vcvarsall.bat', command) + self.assertIn('x86', command) + + command = tools.vcvars_command(settings, arch='x86_64') + self.assertIn('vcvarsall.bat', command) + self.assertIn('amd64', command) + + command = tools.vcvars_command(settings, arch='armv7') + self.assertIn('vcvarsall.bat', command) + self.assertNotIn('arm64', command) + self.assertIn('arm', command) + + command = tools.vcvars_command(settings, arch='armv8') + self.assertIn('vcvarsall.bat', command) + self.assertIn('arm64', command) + + with self.assertRaises(ConanException): + tools.vcvars_command(settings, arch='mips') diff --git a/conans/test/util/vcvars_store_test.py b/conans/test/util/vcvars_store_test.py new file mode 100644 index 00000000000..8b4aa08454f --- /dev/null +++ b/conans/test/util/vcvars_store_test.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import platform +import unittest +from nose.plugins.attrib import attr +from conans.model.settings import Settings +from conans.client.conf import default_settings_yml +from conans.errors import ConanException +from conans import tools + + +@attr('visual_studio') +class VCVarsStoreTest(unittest.TestCase): + def test_81(self): + if platform.system() != "Windows": + return + + settings = Settings.loads(default_settings_yml) + settings.compiler = 'Visual Studio' + settings.compiler.version = '14' + settings.arch = 'x86' + settings.os = 'WindowsStore' + settings.os.version = '8.1' + + command = tools.vcvars_command(settings) + self.assertIn('vcvarsall.bat', command) + self.assertIn('x86', command) + self.assertIn('store', command) + self.assertIn('8.1', command) + + def test_10(self): + if platform.system() != "Windows": + return + sdk_version = tools.find_windows_10_sdk() + if not sdk_version: + return + + settings = Settings.loads(default_settings_yml) + settings.compiler = 'Visual Studio' + settings.compiler.version = '14' + settings.arch = 'x86' + settings.os = 'WindowsStore' + settings.os.version = '10.0' + + command = tools.vcvars_command(settings) + self.assertIn('vcvarsall.bat', command) + self.assertIn('x86', command) + self.assertIn('store', command) + self.assertIn(sdk_version, command) + + def test_invalid(self): + if platform.system() != "Windows": + return + + fake_settings_yml = """ + os: + WindowsStore: + version: ["666"] + arch: [x86] + compiler: + Visual Studio: + runtime: [MD, MT, MTd, MDd] + version: ["8", "9", "10", "11", "12", "14", "15"] + + build_type: [None, Debug, Release] + """ + + settings = Settings.loads(fake_settings_yml) + settings.compiler = 'Visual Studio' + settings.compiler.version = '14' + settings.arch = 'x86' + settings.os = 'WindowsStore' + settings.os.version = '666' + + with self.assertRaises(ConanException): + tools.vcvars_command(settings) From c6fe5b21a92b71128302f2896fb071864ddd6785 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Tue, 21 Nov 2017 05:16:14 +0700 Subject: [PATCH 36/38] Lock fix (#2028) * - handle Ctrl-Break on Windows as well * - fix hangs by tracking PIDs of locking processes * - fixing test failures * changing PIDs approach --- conans/client/command.py | 9 ++++++-- conans/requirements.txt | 2 +- conans/util/locks.py | 47 ++++++++++++++++++++++++++++++---------- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/conans/client/command.py b/conans/client/command.py index 8f158371358..ab53342bc85 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -1261,11 +1261,16 @@ def main(args): try: import signal - def sigint_handler(_, __): + def ctrl_c_handler(_, __): print('You pressed Ctrl+C!') sys.exit(0) - signal.signal(signal.SIGINT, sigint_handler) + def ctrl_break_handler(_, __): + print('You pressed Ctrl+Break!') + sys.exit(0) + + signal.signal(signal.SIGINT, ctrl_c_handler) + signal.signal(signal.SIGBREAK, ctrl_break_handler) error = command.run(args) finally: os.chdir(current_dir) diff --git a/conans/requirements.txt b/conans/requirements.txt index 7a3148eb7ca..fe4873a38e1 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -10,4 +10,4 @@ distro>=1.0.2, <1.1.0 pylint>=1.6.5, <=1.8.0 future==0.16.0 pygments>=2.0, <3.0 - +psutil>=5.4.1 diff --git a/conans/util/locks.py b/conans/util/locks.py index 59a465655a8..fcfc25e6696 100644 --- a/conans/util/locks.py +++ b/conans/util/locks.py @@ -2,6 +2,8 @@ from conans.util.log import logger import time from conans.util.files import save, load +import psutil +import os class NoLock(object): @@ -46,29 +48,50 @@ def _info_locked(self): self._output.info("If not the case, quit, and do 'conan remove %s -f'" % str(self._locked_item)) - def _readers(self): + def _pids(self): try: - return int(load(self._count_file)) + contents = load(self._count_file) except IOError: - return 0 + return [] + else: + if not contents: + return [] + pids = [int(i) for i in contents.split(',')] + valid_pids = [] + for pid in pids: + if not psutil.pid_exists(abs(pid)): + lock_type = "write" if pid < 0 else "read" + self._output.info("invalidate %s lock from PID %s" % (lock_type, abs(pid))) + else: + valid_pids.append(pid) + return valid_pids class ReadLock(Lock): + def _save_pids(self, pids): + save(self._count_file, ','.join(str(pid) for pid in pids)) + def __enter__(self): while True: with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - readers = self._readers() - if readers >= 0: - save(self._count_file, str(readers + 1)) + pids = self._pids() + if len(pids) >= 0: + pids.append(os.getpid()) + save(self._count_file, ','.join(str(pid) for pid in pids)) break self._info_locked() time.sleep(READ_BUSY_DELAY) def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - readers = self._readers() - save(self._count_file, str(readers - 1)) + pids = self._pids() + try: + pids.remove(os.getpid()) + except ValueError: + # legal situation - conan may remove directory with lock file during write + pass + self._save_pids(pids) class WriteLock(Lock): @@ -76,13 +99,13 @@ class WriteLock(Lock): def __enter__(self): while True: with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - readers = self._readers() - if readers == 0: - save(self._count_file, "-1") + pids = self._pids() + if not pids: + save(self._count_file, "-%s" % os.getpid()) break self._info_locked() time.sleep(WRITE_BUSY_DELAY) def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - save(self._count_file, "0") + save(self._count_file, "") From 8d6836f9a4247255b16c234958b8bc0acba1a7a8 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Date: Tue, 21 Nov 2017 08:32:36 +0100 Subject: [PATCH 37/38] Bumped version --- conans/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index 01a6e25d2b2..d74151f3611 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -16,4 +16,4 @@ COMPLEX_SEARCH_CAPABILITY = "complex_search" SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.29.0-dev' +__version__ = '0.29.0' From 6242dc29fb6077ba0ef168b494cc9eab0d974c13 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Date: Tue, 21 Nov 2017 09:35:30 +0100 Subject: [PATCH 38/38] SIBREAK only windows --- conans/client/command.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conans/client/command.py b/conans/client/command.py index ab53342bc85..194f7c85ad5 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -1270,7 +1270,8 @@ def ctrl_break_handler(_, __): sys.exit(0) signal.signal(signal.SIGINT, ctrl_c_handler) - signal.signal(signal.SIGBREAK, ctrl_break_handler) + if sys.platform == 'win32': + signal.signal(signal.SIGBREAK, ctrl_break_handler) error = command.run(args) finally: os.chdir(current_dir)