Add list of depend-only packages to the ebuild generator. (ros-infras…

…tructure#84)

* Add list of depend-only packages to the ebuild generator.

* Simplify commit logic.

* Use superflore.utils print functions.

* Add missing exit call and remove unused variable.

* Remove unused function, and simplify the commit_changes function logic.

* Added the `--only` flag to the readme.

* Address comments by @tfoote.

* Correctly map the distro choice.

* Add missing paren.

* Fix the regression.

* Add regression fix to open embedded.

* Linter updated, this fixes new errors.

README.md

@@ -32,6 +32,7 @@ $ superflore-gen-ebuilds --help
 usage: Deploy ROS packages into Gentoo Linux [-h] [--ros-distro ROS_DISTRO]
                                              [--all] [--dry-run] [--pr-only]
                                              [--output-repository-path OUTPUT_REPOSITORY_PATH]
+                                             [--only ONLY [ONLY ...]]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -42,6 +43,8 @@ optional arguments:
   --pr-only             ONLY file a PR to remote
   --output-repository-path OUTPUT_REPOSITORY_PATH
                         location of the Git repo
+  --only ONLY [ONLY ...]
+                        generate only the specified packages
 ```
 
 Common Usage:
@@ -60,6 +63,8 @@ If you don't want to file a PR with ros/ros-overlay, you should add
 the `--dry-run` flag. You can later decide to file the PR after inspection
 by using the `--pr-only` flag.
 
+To regenerate only the specified packages, use the `--only [pkg1] [pkg2] ... [pkgn]` flag (**note:** you will need to also use the `--ros-distro [distro]` flag.
+
 *If you want to use an existing repo instead of cloning one,
 add `--output-repository-path [path]`.*
 

superflore/TempfileManager.py

@@ -1,5 +1,5 @@
 # Copyright 2017 Open Source Robotics Foundation
-
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

superflore/docker.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import docker
+from superflore.utils import info
+from superflore.utils import ok
 
 
 class Docker(object):
@@ -42,17 +44,16 @@ def run(self, rm=True, show_cmd=False):
             if i != len(self.bash_cmds) - 1:
                 cmd_string += ' && '
         cmd_string += "'"
-
         msg = "Running container with command string '%s'..."
-        print(msg % cmd_string)
+        info(msg % cmd_string)
 
         self.client.containers.run(
             image=self.image,
             remove=rm,
             command=cmd_string,
             volumes=self.directory_map,
         )
-        print("Done!")
+        ok("Docker container exited.")
 
 
 class BuildException(Exception):

superflore/generate_installers.py

@@ -14,7 +14,6 @@
 
 from rosinstall_generator.distro import get_distro
 from rosinstall_generator.distro import get_package_names
-
 from superflore.utils import err
 from superflore.utils import get_pkg_version
 from superflore.utils import info

superflore/generators/bitbake/gen_packages.py

@@ -18,21 +18,16 @@
 from rosdistro.dependency_walker import DependencyWalker
 from rosdistro.manifest_provider import get_release_tag
 from rosdistro.rosdistro import RosPackage
-
 from rosinstall_generator.distro import _generate_rosinstall
 from rosinstall_generator.distro import get_package_names
-
 from superflore.exceptions import NoPkgXml
 from superflore.exceptions import UnresolvedDependency
-
 from superflore.generators.bitbake.yocto_recipe import yoctoRecipe
-
 from superflore.utils import err
 from superflore.utils import get_pkg_version
 from superflore.utils import make_dir
 from superflore.utils import ok
 from superflore.utils import warn
-
 import xmltodict
 
 org = "Open Source Robotics Foundation"

superflore/generators/bitbake/ros_meta.py

@@ -15,7 +15,6 @@
 import time
 
 from superflore.repo_instance import RepoInstance
-
 from superflore.utils import info
 from superflore.utils import rand_ascii_str
 

superflore/generators/bitbake/run.py

@@ -13,15 +13,13 @@
 # limitations under the License.
 
 import argparse
+import os
 import sys
 
 from superflore.generate_installers import generate_installers
-
 from superflore.generators.bitbake.gen_packages import regenerate_installer
 from superflore.generators.bitbake.ros_meta import RosMeta
-
 from superflore.TempfileManager import TempfileManager
-
 from superflore.utils import err
 from superflore.utils import info
 from superflore.utils import ok
@@ -72,6 +70,9 @@ def main():
 
     args = parser.parse_args(sys.argv[1:])
     with TempfileManager(args.output_repository_path) as _repo:
+        if not args.output_repository_path:
+            # give our group write permissions to the temp dir
+            os.chmod(_repo, 17407)
         overlay = RosMeta(_repo, not args.output_repository_path)
         selected_targets = active_distros
         if args.all:

superflore/generators/bitbake/yocto_recipe.py

@@ -29,7 +29,6 @@
 import tarfile
 
 from superflore.exceptions import NoPkgXml
-
 from superflore.utils import get_pkg_version
 from superflore.utils import info
 from superflore.utils import resolve_dep

superflore/generators/ebuild/ebuild.py

@@ -20,6 +20,13 @@
 from superflore.utils import sanitize_string
 from superflore.utils import trim_string
 
+# TODO(allenh1): is there a better way to get these?
+depend_only_pkgs = [
+    'dev-util/gperf',
+    'app-doc/doxygen',
+    'virtual/pkgconfig'
+]
+
 
 class ebuild_keyword(object):
     def __init__(self, arch, stable):
@@ -70,7 +77,9 @@ def add_build_depend(self, depend, internal=True):
             self.depends_external.append(depend)
 
     def add_run_depend(self, rdepend, internal=True):
-        if internal:
+        if rdepend in depend_only_pkgs and not internal:
+            self.depends_external.append(rdepend)
+        elif internal:
             self.rdepends.append(rdepend)
         else:
             self.rdepends_external.append(rdepend)
@@ -164,7 +173,11 @@ def get_ebuild_text(self, distributor, license_text, die_msg=None):
         for rdep in sorted(self.rdepends_external):
             try:
                 for res in resolve_dep(rdep, 'gentoo')[0]:
-                    ret += "    " + res + "\n"
+                    if res in depend_only_pkgs:
+                        self.depends_external.append(rdep)
+                        break
+                    else:
+                        ret += "    " + res + "\n"
             except UnresolvedDependency:
                 self.unresolved_deps.append(rdep)
 

superflore/generators/ebuild/gen_packages.py

@@ -19,21 +19,16 @@
 from rosdistro.dependency_walker import DependencyWalker
 from rosdistro.manifest_provider import get_release_tag
 from rosdistro.rosdistro import RosPackage
-
 from rosinstall_generator.distro import _generate_rosinstall
 from rosinstall_generator.distro import get_package_names
-
 from superflore.exceptions import UnresolvedDependency
-
 from superflore.generators.ebuild.ebuild import Ebuild
 from superflore.generators.ebuild.metadata_xml import metadata_xml
-
 from superflore.utils import err
 from superflore.utils import get_pkg_version
 from superflore.utils import make_dir
 from superflore.utils import ok
 from superflore.utils import warn
-
 import xmltodict
 
 # TODO(allenh1): This is a blacklist of things that

superflore/generators/ebuild/overlay_instance.py

@@ -16,43 +16,31 @@
 import time
 
 from superflore.docker import Docker
-
 from superflore.repo_instance import RepoInstance
-
 from superflore.utils import info
 from superflore.utils import rand_ascii_str
 
 
 class RosOverlay(object):
     def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'):
-        self.repo = RepoInstance(org, repo, repo_dir, do_clone)
+        self.repo = RepoInstance(
+            org, repo, repo_dir=repo_dir, do_clone=do_clone
+        )
         self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
         info('Creating new branch {0}...'.format(self.branch_name))
         self.repo.create_branch(self.branch_name)
 
-    def clean_ros_ebuild_dirs(self, distro=None):
-        if distro:
-            info('Cleaning up ros-{0} directory...'.format(distro))
-            self.repo.git.rm('-rf', 'ros-{0}'.format(distro))
-        else:
-            info('Cleaning up ros-* directories...')
-            self.repo.git.rm('-rf', 'ros-*')
-
     def commit_changes(self, distro):
         info('Adding changes...')
-        if distro:
-            self.repo.git.add('ros-{0}'.format(distro))
-        else:
-            self.repo.git.add('ros-*')
-            distro = 'update'
-        commit_msg = {
-            'update': 'rosdistro sync, {0}',
-            'all': 'regenerate all distros, {0}',
-            'lunar': 'regenerate ros-lunar, {0}',
-            'indigo': 'regenerate ros-indigo, {0}',
-            'kinetic': 'regenerate ros-kinetic, {0}',
-        }[distro].format(time.ctime())
+        self.repo.git.add(self.repo.repo_dir)
         info('Committing to branch {0}...'.format(self.branch_name))
+        commit_msg = {
+            'update': 'rosdistro sync, ',
+            'all': 'regenerate all distros, ',
+            'lunar': 'regenerate ros-lunar, ',
+            'indigo': 'regenerate ros-indigo, ',
+            'kinetic': 'regenerate ros-kinetic, ',
+        }[distro or 'update'] + time.ctime()
         self.repo.git.commit(m='{0}'.format(commit_msg))
 
     def regenerate_manifests(self, regen_dict):

superflore/generators/ebuild/run.py

@@ -19,16 +19,11 @@
 import time
 
 from rosinstall_generator.distro import get_distro
-
 from superflore.generate_installers import generate_installers
-
 from superflore.generators.ebuild.gen_packages import regenerate_pkg
 from superflore.generators.ebuild.overlay_instance import RosOverlay
-
 from superflore.repo_instance import RepoInstance
-
 from superflore.TempfileManager import TempfileManager
-
 from superflore.utils import err
 from superflore.utils import info
 from superflore.utils import ok
@@ -102,10 +97,9 @@ def main():
         selected_targets = [args.ros_distro]
         preserve_existing = False
     elif args.dry_run and args.pr_only:
-        err('Invalid args! cannot dry-run and file PR')
-        sys.exit(1)
+        parser.error('Invalid args! cannot dry-run and file PR')
     elif args.pr_only and not args.output_repository_path:
-        err('Invalid args! no repository specified')
+        parser.error('Invalid args! no repository specified')
     elif args.pr_only:
         try:
             with open('.pr-message.tmp', 'r') as msg_file:
@@ -132,10 +126,14 @@ def main():
             err('Failed to file PR!')
             err('reason: {0}'.format(e))
             sys.exit(1)
+    if not selected_targets:
+        selected_targets = active_distros
     with TempfileManager(args.output_repository_path) as _repo:
+        if not args.output_repository_path:
+            # give our group write permissions to the temp dir
+            os.chmod(_repo, 17407)
         # clone if args.output_repository_path is None
         overlay = RosOverlay(_repo, not args.output_repository_path)
-        selected_targets = active_distros
         # generate installers
         total_installers = dict()
         total_broken = set()
@@ -146,25 +144,24 @@ def main():
                 info("Regenerating package '%s'..." % pkg)
                 regenerate_pkg(
                     overlay,
-                    pkg=pkg,
-                    distro=get_distro(args.ros_distro)
+                    pkg,
+                    get_distro(args.ros_distro),
+                    preserve_existing
                 )
             # Commit changes and file pull request
             regen_dict = dict()
             regen_dict[args.ros_distro] = args.only
             overlay.regenerate_manifests(regen_dict)
             overlay.commit_changes(args.ros_distro)
             delta = "Regenerated: '%s'\n" % args.only
-            missing_deps = ''
             if args.dry_run:
                 info('Running in dry mode, not filing PR')
                 title_file = open('.pr-title.tmp', 'w')
                 title_file.write('rosdistro sync, {0}\n'.format(time.ctime()))
                 pr_message_file = open('.pr-message.tmp', 'w')
-                pr_message_file.write('%s\n%s\n' % (delta, missing_deps))
+                pr_message_file.write('%s\n%s\n' % (delta, ''))
                 sys.exit(0)
-            file_pr(overlay, delta, missing_deps)
-
+            file_pr(overlay, delta, '')
             clean_up()
             ok('Successfully synchronized repositories!')
             sys.exit(0)

superflore/repo_instance.py

@@ -16,8 +16,7 @@
 
 from git import Repo
 from git.exc import GitCommandError as GitGotGot
-
-from superflore.utils import err as error
+from superflore.utils import err
 from superflore.utils import info
 from superflore.utils import ok
 
@@ -54,8 +53,8 @@ def remove_file(self, filename, ignore_fail=False):
                 return
             fail_msg = 'Failed to remove file {0}'.format(filename)
             fail_msg += 'from source control.'
-            error(fail_msg)
-            error(' Exception: {0}'.format(g))
+            err(fail_msg)
+            err(' Exception: {0}'.format(g))
 
     def create_branch(self, branch_name):
         """

superflore/rosdep_support.py

@@ -33,7 +33,6 @@
 from rosdep2 import create_default_installer_context
 from rosdep2.catkin_support import get_catkin_view
 from rosdep2.lookup import ResolutionError
-
 from superflore.exceptions import UnresolvedDependency
 
 DEFAULT_ROS_DISTRO = 'indigo'

superflore/utils.py

@@ -21,11 +21,8 @@
 
 from superflore.exceptions import UnknownLicense
 from superflore.exceptions import UnknownPlatform
-
 from superflore.rosdep_support import resolve_rosdep_key
-
 from termcolor import colored
-
 import yaml
 
 if sys.version_info[0] == 2: