diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..a81c8ee1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/bin/colcon b/bin/colcon index 1d7cb0d2..cc040565 100755 --- a/bin/colcon +++ b/bin/colcon @@ -67,12 +67,14 @@ from colcon_core.shell.bat import BatShell # noqa: E402 from colcon_core.shell.dsv import DsvShell # noqa: E402 from colcon_core.shell.sh import ShShell # noqa: E402 from colcon_core.task.python.build import PythonBuildTask # noqa: E402 +from colcon_core.task.python.stage import PythonStageTask # noqa: E402 from colcon_core.task.python.test import PythonTestTask # noqa: E402 from colcon_core.task.python.test.pytest \ import PytestPythonTestingStep # noqa: E402 from colcon_core.task.python.test.setuppy_test \ import SetuppyPythonTestingStep # noqa: E402 from colcon_core.verb.build import BuildVerb # noqa: E402 +from colcon_core.verb.stage import StageVerb # noqa: E402 from colcon_core.verb.test import TestVerb # noqa: E402 @@ -127,11 +129,15 @@ custom_entry_points.update({ 'colcon_core.task.build': { 'python': PythonBuildTask, }, + 'colcon_core.task.stage': { + 'python': PythonStageTask, + }, 'colcon_core.task.test': { 'python': PythonTestTask, }, 'colcon_core.verb': { 'build': BuildVerb, + 'stage': StageVerb, 'test': TestVerb, }, }) diff --git a/colcon_core/task/python/stage.py b/colcon_core/task/python/stage.py new file mode 100644 index 00000000..2f315311 --- /dev/null +++ b/colcon_core/task/python/stage.py @@ -0,0 +1,77 @@ +# Copyright 2016-2018 Dirk Thomas +# Copyright 2021 Ruffin White +# Licensed under the Apache License, Version 2.0 + +from contextlib import suppress +# with suppress(ImportError): +# # needed before importing distutils +# # to avoid warning introduced in setuptools 49.2.0 +# import setuptools # noqa: F401 +# from distutils.sysconfig import get_python_lib +# import locale +import os +from pathlib import Path +# import shutil +# import sys +# from sys import executable + +# from colcon_core.environment import create_environment_hooks +# from colcon_core.environment import create_environment_scripts +from colcon_core.logging import colcon_logger +from colcon_core.plugin_system import satisfies_version +# from colcon_core.shell import create_environment_hook +# from colcon_core.shell import get_command_environment +# from colcon_core.subprocess import check_output +# from colcon_core.task import run +from colcon_core.task import TaskExtensionPoint +# from colcon_core.task.python import get_data_files_mapping +# from colcon_core.task.python import get_setup_data +from dirhash import dirhash + +logger = colcon_logger.getChild(__name__) + + +class PythonStageTask(TaskExtensionPoint): + """Stage Python packages.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version(TaskExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + async def stage(self, *, additional_hooks=None): # noqa: D102 + pkg = self.context.pkg + args = self.context.args + + logger.info( + "Staging Python package in '{args.path}'".format_map(locals())) + + # Use the number of CPU cores + jobs = os.cpu_count() + with suppress(AttributeError): + # consider restricted set of CPUs if applicable + jobs = min(jobs, len(os.sched_getaffinity(0))) + if jobs is None: + # the number of cores can't be determined + jobs = 1 + + # ignore all . files and . folders + current_checksum = dirhash(args.path, 'md5', ignore=['.*'], jobs=jobs) + + # os.makedirs(args.build_base, exist_ok=True) + stage_base = Path(args.build_base, 'stage') + stage_base.mkdir(parents=True, exist_ok=True) + current_path = Path(stage_base, 'colcon_stage_current.txt') + previous_path = Path(stage_base, 'colcon_stage_previous.txt') + + current_path.write_text(str(current_checksum) + '\n') + + previous_checksum = None + if previous_path.exists(): + previous_checksum = previous_path.read_text().rstrip() + + if args.tare_changes: + previous_path.write_text(str(current_checksum) + '\n') + return 0 + elif previous_checksum == current_checksum: + return 0 + return 'changed' diff --git a/colcon_core/verb/stage.py b/colcon_core/verb/stage.py new file mode 100644 index 00000000..87acf8c5 --- /dev/null +++ b/colcon_core/verb/stage.py @@ -0,0 +1,207 @@ +# Copyright 2016-2018 Dirk Thomas +# Copyright 2021 Ruffin White +# Licensed under the Apache License, Version 2.0 + +from collections import OrderedDict +import os +import os.path +from pathlib import Path +# import traceback + +from colcon_core.argument_parser.destination_collector \ + import DestinationCollectorDecorator +from colcon_core.event.job import JobUnselected +from colcon_core.event_handler import add_event_handler_arguments +from colcon_core.executor import add_executor_arguments +from colcon_core.executor import execute_jobs +from colcon_core.executor import Job +from colcon_core.executor import OnError +from colcon_core.package_identification.ignore import IGNORE_MARKER +from colcon_core.package_selection import add_arguments \ + as add_packages_arguments +from colcon_core.package_selection import get_packages +from colcon_core.plugin_system import satisfies_version +# from colcon_core.shell import get_shell_extensions +from colcon_core.task import add_task_arguments +from colcon_core.task import get_task_extension +from colcon_core.task import TaskContext +from colcon_core.verb import check_and_mark_build_tool +from colcon_core.verb import check_and_mark_install_layout +from colcon_core.verb import logger +from colcon_core.verb import update_object +from colcon_core.verb import VerbExtensionPoint + + +class StagePackageArguments: + """Arguments to stage a specific package.""" + + def __init__(self, pkg, args, *, additional_destinations=None): + """ + Construct a StagePackageArguments. + + :param pkg: The package descriptor + :param args: The parsed command line arguments + :param list additional_destinations: The destinations of additional + arguments + """ + super().__init__() + self.path = os.path.abspath( + os.path.join(os.getcwd(), str(pkg.path))) + self.build_base = os.path.abspath(os.path.join( + os.getcwd(), args.build_base, pkg.name)) + self.install_base = os.path.abspath(os.path.join( + os.getcwd(), args.install_base)) + if not args.merge_install: + self.install_base = os.path.join( + self.install_base, pkg.name) + self.test_result_base = os.path.abspath(os.path.join( + os.getcwd(), args.test_result_base, pkg.name)) \ + if args.test_result_base else None + self.tare_changes = args.tare_changes + + # set additional arguments + for dest in (additional_destinations or []): + # from the command line + if hasattr(args, dest): + update_object( + self, dest, getattr(args, dest), + pkg.name, 'stage', 'command line') + # from the package metadata + if dest in pkg.metadata: + update_object( + self, dest, pkg.metadata[dest], + pkg.name, 'stage', 'package metadata') + + +class StageVerb(VerbExtensionPoint): + """Stage a set of packages.""" + + def __init__(self): # noqa: D107 + super().__init__() + satisfies_version(VerbExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + def add_arguments(self, *, parser): # noqa: D102 + parser.add_argument( + '--build-base', + default='build', + help='The base path for all build directories (default: build)') + parser.add_argument( + '--install-base', + default='install', + help='The base path for all install prefixes (default: install)') + parser.add_argument( + '--merge-install', + action='store_true', + help='Merge all install prefixes into a single location') + parser.add_argument( + '--test-result-base', + help='The base path for all test results (default: --build-base)') + parser.add_argument( + '--tare-changes', + action='store_true', + help='Tare changes by resets reference checksums for staging') + add_executor_arguments(parser) + add_event_handler_arguments(parser) + + add_packages_arguments(parser) + + decorated_parser = DestinationCollectorDecorator(parser) + add_task_arguments(decorated_parser, 'colcon_core.task.build') + self.task_argument_destinations = decorated_parser.get_destinations() + + def main(self, *, context): # noqa: D102 + check_and_mark_build_tool(context.args.build_base) + check_and_mark_install_layout( + context.args.install_base, + merge_install=context.args.merge_install) + + self._create_paths(context.args) + + decorators = get_packages( + context.args, + additional_argument_names=self.task_argument_destinations, + recursive_categories=('run', )) + + install_base = os.path.abspath(os.path.join( + os.getcwd(), context.args.install_base)) + jobs, unselected_packages = self._get_jobs( + context.args, decorators, install_base) + + # TODO: OnError.continue_ is a workaround given rc need not be 0 + # on_error = OnError.interrupt \ + # if not context.args.continue_on_error else OnError.skip_downstream + on_error = OnError.continue_ + + def post_unselected_packages(*, event_queue): + nonlocal unselected_packages + names = [pkg.name for pkg in unselected_packages] + for name in sorted(names): + event_queue.put( + (JobUnselected(name), None)) + + rc = execute_jobs( + context, jobs, on_error=on_error, + pre_execution_callback=post_unselected_packages) + + return rc + + def _create_paths(self, args): + self._create_path(args.build_base) + self._create_path(args.install_base) + + def _create_path(self, path): + path = Path(os.path.abspath(path)) + if not path.exists(): + path.mkdir(parents=True, exist_ok=True) + ignore_marker = path / IGNORE_MARKER + if not os.path.lexists(str(ignore_marker)): + with ignore_marker.open('w'): + pass + + def _get_jobs(self, args, decorators, install_base): + jobs = OrderedDict() + unselected_packages = set() + for decorator in decorators: + pkg = decorator.descriptor + + if not decorator.selected: + unselected_packages.add(pkg) + continue + + # TODO: workaround by hard coding pkg.type to default for entry point + # extension = get_task_extension('colcon_core.task.stage', pkg.type) + extension = get_task_extension('colcon_core.task.stage', 'default') + if not extension: + logger.warning( + "No task extension to 'stage' a '{pkg.type}' package" + .format_map(locals())) + continue + + recursive_dependencies = OrderedDict() + for dep_name in decorator.recursive_dependencies: + dep_path = install_base + if not args.merge_install: + dep_path = os.path.join(dep_path, dep_name) + recursive_dependencies[dep_name] = dep_path + + package_args = StagePackageArguments( + pkg, args, additional_destinations=self + .task_argument_destinations.values()) + ordered_package_args = ', '.join([ + ('%s: %s' % (repr(k), repr(package_args.__dict__[k]))) + for k in sorted(package_args.__dict__.keys()) + ]) + logger.debug( + "Staging package '{pkg.name}' with the following arguments: " + '{{{ordered_package_args}}}'.format_map(locals())) + task_context = TaskContext( + pkg=pkg, args=package_args, + dependencies=recursive_dependencies) + + job = Job( + identifier=pkg.name, + dependencies=set(recursive_dependencies.keys()), + task=extension, task_context=task_context) + + jobs[pkg.name] = job + return jobs, unselected_packages diff --git a/setup.cfg b/setup.cfg index 9d07142c..766bd253 100644 --- a/setup.cfg +++ b/setup.cfg @@ -99,6 +99,7 @@ colcon_core.extension_point = colcon_core.python_testing = colcon_core.task.python.test:PythonTestingStepExtensionPoint colcon_core.shell = colcon_core.shell:ShellExtensionPoint colcon_core.task.build = colcon_core.task:TaskExtensionPoint + colcon_core.task.stage = colcon_core.task:TaskExtensionPoint colcon_core.task.test = colcon_core.task:TaskExtensionPoint colcon_core.verb = colcon_core.verb:VerbExtensionPoint colcon_core.package_augmentation = @@ -120,10 +121,13 @@ colcon_core.shell = sh = colcon_core.shell.sh:ShShell colcon_core.task.build = python = colcon_core.task.python.build:PythonBuildTask +colcon_core.task.stage = + default = colcon_core.task.python.stage:PythonStageTask colcon_core.task.test = python = colcon_core.task.python.test:PythonTestTask colcon_core.verb = build = colcon_core.verb.build:BuildVerb + stage = colcon_core.verb.stage:StageVerb test = colcon_core.verb.test:TestVerb console_scripts = colcon = colcon_core.command:main diff --git a/test/spell_check.words b/test/spell_check.words index b420f7cb..f3391c2d 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -11,6 +11,8 @@ callables capsys catched changelog +checksum +checksums classname colcon coloredlogs @@ -28,6 +30,7 @@ defaultdict depreated deps descs +dirhash distlib docstring executables @@ -35,6 +38,7 @@ exitstatus fdopen filterwarnings functools +getaffinity getcategory getpid getpreferredencoding