diff --git a/README.md b/README.md index 5a6e22ba..e5abed57 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ $ superflore-gen-ebuilds --help usage: Deploy ROS packages into Gentoo Linux [-h] [--ros-distro ROS_DISTRO] [--all] [--dry-run] [--pr-only] [--output-repository-path OUTPUT_REPOSITORY_PATH] + [--only ONLY [ONLY ...]] optional arguments: -h, --help show this help message and exit @@ -42,6 +43,8 @@ optional arguments: --pr-only ONLY file a PR to remote --output-repository-path OUTPUT_REPOSITORY_PATH location of the Git repo + --only ONLY [ONLY ...] + generate only the specified packages ``` Common Usage: @@ -60,6 +63,8 @@ If you don't want to file a PR with ros/ros-overlay, you should add the `--dry-run` flag. You can later decide to file the PR after inspection by using the `--pr-only` flag. +To regenerate only the specified packages, use the `--only [pkg1] [pkg2] ... [pkgn]` flag (**note:** you will need to also use the `--ros-distro [distro]` flag. + *If you want to use an existing repo instead of cloning one, add `--output-repository-path [path]`.* diff --git a/superflore/TempfileManager.py b/superflore/TempfileManager.py index 248da1a4..2459fdf3 100644 --- a/superflore/TempfileManager.py +++ b/superflore/TempfileManager.py @@ -1,5 +1,5 @@ # Copyright 2017 Open Source Robotics Foundation - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/superflore/docker.py b/superflore/docker.py index 02cd113c..26430080 100644 --- a/superflore/docker.py +++ b/superflore/docker.py @@ -13,6 +13,8 @@ # limitations under the License. import docker +from superflore.utils import info +from superflore.utils import ok class Docker(object): @@ -42,9 +44,8 @@ def run(self, rm=True, show_cmd=False): if i != len(self.bash_cmds) - 1: cmd_string += ' && ' cmd_string += "'" - msg = "Running container with command string '%s'..." - print(msg % cmd_string) + info(msg % cmd_string) self.client.containers.run( image=self.image, @@ -52,7 +53,7 @@ def run(self, rm=True, show_cmd=False): command=cmd_string, volumes=self.directory_map, ) - print("Done!") + ok("Docker container exited.") class BuildException(Exception): diff --git a/superflore/generate_installers.py b/superflore/generate_installers.py index 81bca8c4..8efaa5d1 100644 --- a/superflore/generate_installers.py +++ b/superflore/generate_installers.py @@ -14,7 +14,6 @@ from rosinstall_generator.distro import get_distro from rosinstall_generator.distro import get_package_names - from superflore.utils import err from superflore.utils import get_pkg_version from superflore.utils import info diff --git a/superflore/generators/bitbake/gen_packages.py b/superflore/generators/bitbake/gen_packages.py index ff9180cb..b1837210 100644 --- a/superflore/generators/bitbake/gen_packages.py +++ b/superflore/generators/bitbake/gen_packages.py @@ -18,21 +18,16 @@ from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag from rosdistro.rosdistro import RosPackage - from rosinstall_generator.distro import _generate_rosinstall from rosinstall_generator.distro import get_package_names - from superflore.exceptions import NoPkgXml from superflore.exceptions import UnresolvedDependency - from superflore.generators.bitbake.yocto_recipe import yoctoRecipe - from superflore.utils import err from superflore.utils import get_pkg_version from superflore.utils import make_dir from superflore.utils import ok from superflore.utils import warn - import xmltodict org = "Open Source Robotics Foundation" diff --git a/superflore/generators/bitbake/ros_meta.py b/superflore/generators/bitbake/ros_meta.py index 043703bc..e140ba6b 100644 --- a/superflore/generators/bitbake/ros_meta.py +++ b/superflore/generators/bitbake/ros_meta.py @@ -15,7 +15,6 @@ import time from superflore.repo_instance import RepoInstance - from superflore.utils import info from superflore.utils import rand_ascii_str diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index ce847b1c..62a31820 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -13,15 +13,13 @@ # limitations under the License. import argparse +import os import sys from superflore.generate_installers import generate_installers - from superflore.generators.bitbake.gen_packages import regenerate_installer from superflore.generators.bitbake.ros_meta import RosMeta - from superflore.TempfileManager import TempfileManager - from superflore.utils import err from superflore.utils import info from superflore.utils import ok @@ -72,6 +70,9 @@ def main(): args = parser.parse_args(sys.argv[1:]) with TempfileManager(args.output_repository_path) as _repo: + if not args.output_repository_path: + # give our group write permissions to the temp dir + os.chmod(_repo, 17407) overlay = RosMeta(_repo, not args.output_repository_path) selected_targets = active_distros if args.all: diff --git a/superflore/generators/bitbake/yocto_recipe.py b/superflore/generators/bitbake/yocto_recipe.py index 7a0d19e9..82184d63 100644 --- a/superflore/generators/bitbake/yocto_recipe.py +++ b/superflore/generators/bitbake/yocto_recipe.py @@ -29,7 +29,6 @@ import tarfile from superflore.exceptions import NoPkgXml - from superflore.utils import get_pkg_version from superflore.utils import info from superflore.utils import resolve_dep diff --git a/superflore/generators/ebuild/ebuild.py b/superflore/generators/ebuild/ebuild.py index f9ff1d72..0ae0995c 100644 --- a/superflore/generators/ebuild/ebuild.py +++ b/superflore/generators/ebuild/ebuild.py @@ -20,6 +20,13 @@ from superflore.utils import sanitize_string from superflore.utils import trim_string +# TODO(allenh1): is there a better way to get these? +depend_only_pkgs = [ + 'dev-util/gperf', + 'app-doc/doxygen', + 'virtual/pkgconfig' +] + class ebuild_keyword(object): def __init__(self, arch, stable): @@ -70,7 +77,9 @@ def add_build_depend(self, depend, internal=True): self.depends_external.append(depend) def add_run_depend(self, rdepend, internal=True): - if internal: + if rdepend in depend_only_pkgs and not internal: + self.depends_external.append(rdepend) + elif internal: self.rdepends.append(rdepend) else: self.rdepends_external.append(rdepend) @@ -164,7 +173,11 @@ def get_ebuild_text(self, distributor, license_text, die_msg=None): for rdep in sorted(self.rdepends_external): try: for res in resolve_dep(rdep, 'gentoo')[0]: - ret += " " + res + "\n" + if res in depend_only_pkgs: + self.depends_external.append(rdep) + break + else: + ret += " " + res + "\n" except UnresolvedDependency: self.unresolved_deps.append(rdep) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index 543ad9e4..2f322039 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -19,21 +19,16 @@ from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag from rosdistro.rosdistro import RosPackage - from rosinstall_generator.distro import _generate_rosinstall from rosinstall_generator.distro import get_package_names - from superflore.exceptions import UnresolvedDependency - from superflore.generators.ebuild.ebuild import Ebuild from superflore.generators.ebuild.metadata_xml import metadata_xml - from superflore.utils import err from superflore.utils import get_pkg_version from superflore.utils import make_dir from superflore.utils import ok from superflore.utils import warn - import xmltodict # TODO(allenh1): This is a blacklist of things that diff --git a/superflore/generators/ebuild/overlay_instance.py b/superflore/generators/ebuild/overlay_instance.py index f1e2dd3c..e58bfb42 100644 --- a/superflore/generators/ebuild/overlay_instance.py +++ b/superflore/generators/ebuild/overlay_instance.py @@ -16,43 +16,31 @@ import time from superflore.docker import Docker - from superflore.repo_instance import RepoInstance - from superflore.utils import info from superflore.utils import rand_ascii_str class RosOverlay(object): def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'): - self.repo = RepoInstance(org, repo, repo_dir, do_clone) + self.repo = RepoInstance( + org, repo, repo_dir=repo_dir, do_clone=do_clone + ) self.branch_name = 'gentoo-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name) - def clean_ros_ebuild_dirs(self, distro=None): - if distro: - info('Cleaning up ros-{0} directory...'.format(distro)) - self.repo.git.rm('-rf', 'ros-{0}'.format(distro)) - else: - info('Cleaning up ros-* directories...') - self.repo.git.rm('-rf', 'ros-*') - def commit_changes(self, distro): info('Adding changes...') - if distro: - self.repo.git.add('ros-{0}'.format(distro)) - else: - self.repo.git.add('ros-*') - distro = 'update' - commit_msg = { - 'update': 'rosdistro sync, {0}', - 'all': 'regenerate all distros, {0}', - 'lunar': 'regenerate ros-lunar, {0}', - 'indigo': 'regenerate ros-indigo, {0}', - 'kinetic': 'regenerate ros-kinetic, {0}', - }[distro].format(time.ctime()) + self.repo.git.add(self.repo.repo_dir) info('Committing to branch {0}...'.format(self.branch_name)) + commit_msg = { + 'update': 'rosdistro sync, ', + 'all': 'regenerate all distros, ', + 'lunar': 'regenerate ros-lunar, ', + 'indigo': 'regenerate ros-indigo, ', + 'kinetic': 'regenerate ros-kinetic, ', + }[distro or 'update'] + time.ctime() self.repo.git.commit(m='{0}'.format(commit_msg)) def regenerate_manifests(self, regen_dict): diff --git a/superflore/generators/ebuild/run.py b/superflore/generators/ebuild/run.py index 20c4f4b7..179e8c31 100755 --- a/superflore/generators/ebuild/run.py +++ b/superflore/generators/ebuild/run.py @@ -19,16 +19,11 @@ import time from rosinstall_generator.distro import get_distro - from superflore.generate_installers import generate_installers - from superflore.generators.ebuild.gen_packages import regenerate_pkg from superflore.generators.ebuild.overlay_instance import RosOverlay - from superflore.repo_instance import RepoInstance - from superflore.TempfileManager import TempfileManager - from superflore.utils import err from superflore.utils import info from superflore.utils import ok @@ -102,10 +97,9 @@ def main(): selected_targets = [args.ros_distro] preserve_existing = False elif args.dry_run and args.pr_only: - err('Invalid args! cannot dry-run and file PR') - sys.exit(1) + parser.error('Invalid args! cannot dry-run and file PR') elif args.pr_only and not args.output_repository_path: - err('Invalid args! no repository specified') + parser.error('Invalid args! no repository specified') elif args.pr_only: try: with open('.pr-message.tmp', 'r') as msg_file: @@ -132,10 +126,14 @@ def main(): err('Failed to file PR!') err('reason: {0}'.format(e)) sys.exit(1) + if not selected_targets: + selected_targets = active_distros with TempfileManager(args.output_repository_path) as _repo: + if not args.output_repository_path: + # give our group write permissions to the temp dir + os.chmod(_repo, 17407) # clone if args.output_repository_path is None overlay = RosOverlay(_repo, not args.output_repository_path) - selected_targets = active_distros # generate installers total_installers = dict() total_broken = set() @@ -146,8 +144,9 @@ def main(): info("Regenerating package '%s'..." % pkg) regenerate_pkg( overlay, - pkg=pkg, - distro=get_distro(args.ros_distro) + pkg, + get_distro(args.ros_distro), + preserve_existing ) # Commit changes and file pull request regen_dict = dict() @@ -155,16 +154,14 @@ def main(): overlay.regenerate_manifests(regen_dict) overlay.commit_changes(args.ros_distro) delta = "Regenerated: '%s'\n" % args.only - missing_deps = '' if args.dry_run: info('Running in dry mode, not filing PR') title_file = open('.pr-title.tmp', 'w') title_file.write('rosdistro sync, {0}\n'.format(time.ctime())) pr_message_file = open('.pr-message.tmp', 'w') - pr_message_file.write('%s\n%s\n' % (delta, missing_deps)) + pr_message_file.write('%s\n%s\n' % (delta, '')) sys.exit(0) - file_pr(overlay, delta, missing_deps) - + file_pr(overlay, delta, '') clean_up() ok('Successfully synchronized repositories!') sys.exit(0) diff --git a/superflore/repo_instance.py b/superflore/repo_instance.py index a8c74238..69c46e14 100644 --- a/superflore/repo_instance.py +++ b/superflore/repo_instance.py @@ -16,8 +16,7 @@ from git import Repo from git.exc import GitCommandError as GitGotGot - -from superflore.utils import err as error +from superflore.utils import err from superflore.utils import info from superflore.utils import ok @@ -54,8 +53,8 @@ def remove_file(self, filename, ignore_fail=False): return fail_msg = 'Failed to remove file {0}'.format(filename) fail_msg += 'from source control.' - error(fail_msg) - error(' Exception: {0}'.format(g)) + err(fail_msg) + err(' Exception: {0}'.format(g)) def create_branch(self, branch_name): """ diff --git a/superflore/rosdep_support.py b/superflore/rosdep_support.py index 56a2e6ef..608503dc 100644 --- a/superflore/rosdep_support.py +++ b/superflore/rosdep_support.py @@ -33,7 +33,6 @@ from rosdep2 import create_default_installer_context from rosdep2.catkin_support import get_catkin_view from rosdep2.lookup import ResolutionError - from superflore.exceptions import UnresolvedDependency DEFAULT_ROS_DISTRO = 'indigo' diff --git a/superflore/utils.py b/superflore/utils.py index 1204c7f1..221e5eba 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -21,11 +21,8 @@ from superflore.exceptions import UnknownLicense from superflore.exceptions import UnknownPlatform - from superflore.rosdep_support import resolve_rosdep_key - from termcolor import colored - import yaml if sys.version_info[0] == 2: