Nvidia 18.04 support (#15)

* Switch behavior of nvida based on 18.04 vs 16.04
Resolves #14

Build on top of the image based os detection and eliminate os detection dependency.

* test nvidia on xenial and bionic
* inject the base_image into the cli args for use in extensions.
* Fix unit tests building of base images
Fixes #25

README.md

@@ -10,14 +10,18 @@ Rocker supports extensions via entry points there are some built in but you can
 
 # Prerequisites
 
-This should work on most systems using with a recent docker version available. 
+This should work on most systems using with a recent docker version available.
+
+Docker installation instructions: https://docs.docker.com/install/
 
 ## NVIDIA settings
 
 For the NVIDIA option this has been demonstrated using Ubuntu 16.04.5 running Kernel 4.15 and nvidia docker2 and the nvidia 384 driver.
 It did not work using the nvidia 340 driver.
 
-It's also expected to work with 18.04 and a recent nvidia driver as well.
+It's also been tested on Ubuntu 18.04 with the 390 Nvidia driver.
+
+Install nvidia-docker 2: https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)
 
 
 # Installation
@@ -27,11 +31,6 @@ Debian packages are available from the ROS repositories. You can set them up in
 
 Then you can `sudo apt-get install python3-rocker`
 
-On Ubuntu older than bionic you will need to install python3-distro manually first. 
-
-See https://github.com/osrf/rocker/blob/master/backport_xenial.bash for tested on xenial.
-
-
 ## PIP
 
 Rocker is available via pip you can install it via pip using
@@ -70,23 +69,22 @@ To run tests install nose and coverage in the venv
 
 Then you can run nosetests.
 
-    nosetests-3.4 --with-coverage --cover-package rocker -s test/
+    nosetests-3.4 --with-coverage --cover-package rocker
 
-NOtes: 
+Notes:
 
-- Make sure to use the python3 one from inside the environment.
-- You also must run with the console output due to [#9](https://github.com/osrf/rocker/issues/9)
+- Make sure to use the python3 instance of nosetest from inside the environment.
 - The tests include an nvidia test which assumes you're using a machine with an nvidia gpu.
 
 
 # Example usage
 
 
-## Fly a iris
+## Fly a drone
 
 Example usage with an iris
 
-    rocker --nvidia --user --pull --pulse tfoote/drone_demo roslaunch sitl_launcher demo.launch mavros:=true gui:=false
+    rocker --nvidia --user --pull --pulse tfoote/drone_demo
 
 After the ekf converges, 
 
@@ -103,4 +101,15 @@ In QGroundControl go ahead and make a mission, upload it, and then start the mis
 
 ## ROS 2 rviz
 
-    rocker --nvidia --net=host osrf/ros:crystal-desktop rviz2
+    rocker --nvidia osrf/ros:crystal-desktop rviz2
+
+
+## Generic gazebo
+
+On Xenial
+
+    rocker --nvidia osrf/ros:kinetic-desktop-full gazebo
+
+On Bionic
+
+    rocker --nvidia osrf/ros:melodic-desktop-full gazebo

setup.py

@@ -4,7 +4,6 @@
 from setuptools import setup
 
 install_requires = [
-    'distro',
     'empy',
     'pexpect',
 ]

src/rocker/cli.py

@@ -58,6 +58,7 @@ def main():
     dig = DockerImageGenerator(active_extensions, args_dict, base_image)
     exit_code = dig.build(**vars(args))
     if exit_code != 0:
+        print("Build failed exiting")
         return exit_code
     # Convert command into string
     args.command = ' '.join(args.command)

src/rocker/core.py

@@ -42,6 +42,7 @@ class DockerImageGenerator(object):
     def __init__(self, active_extensions, cliargs, base_image):
         self.built = False
         self.cliargs = cliargs
+        self.cliargs['base_image'] = base_image # inject base image into arguments for use
         self.active_extensions = active_extensions
 
         self.dockerfile = generate_dockerfile(active_extensions, self.cliargs, base_image)
@@ -81,12 +82,16 @@ def build(self, **kwargs):
                 if success_detected:
                         self.built = True
                         return 0
+                else:
+                    print("no more output and success not detected")
+                    return 2
 
             except docker.errors.APIError as ex:
                 print("Docker build failed\n", ex)
                 print(ex.output)
                 return 1
         # Unknown error 
+        print("Unknown error")
         return 2
 
     def run(self, command='', **kwargs):

src/rocker/extensions.py

@@ -18,10 +18,11 @@
 import getpass
 import pkgutil
 from pathlib import Path
-
 import subprocess
+import sys
 
-import distro
+from .os_detector import build_detector_image
+from .os_detector import detect_os
 
 def name_to_argument(name):
     return '--%s' % name.replace('_', '-')
@@ -60,16 +61,16 @@ def get_name():
         return 'dev_helpers'
 
     def __init__(self):
-        self.env_subs = None
+        self._env_subs = None
         self.name = DevHelpers.get_name()
 
 
     def get_environment_subs(self):
-        if not self.env_subs:
-            self.env_subs = {}
-            self.env_subs['user_id'] = os.getuid()
-            self.env_subs['username'] = getpass.getuser()
-        return self.env_subs
+        if not self._env_subs:
+            self._env_subs = {}
+            self._env_subs['user_id'] = os.getuid()
+            self._env_subs['username'] = getpass.getuser()
+        return self._env_subs
 
     def get_preamble(self, cliargs):
         return ''
@@ -91,37 +92,42 @@ def get_name():
         return 'nvidia'
 
     def __init__(self):
-        self.env_subs = None
+        self._env_subs = None
         self.name = Nvidia.get_name()
         self.xauth = '/tmp/.docker.xauth'
+        self.supported_distros = ['Ubuntu']
         self.supported_versions = ['16.04', '18.04']
 
 
-    def get_environment_subs(self):
-        if not self.env_subs:
-            self.env_subs = {}
-            self.env_subs['user_id'] = os.getuid()
-            self.env_subs['username'] = getpass.getuser()
-            self.env_subs['DISPLAY'] = os.getenv('DISPLAY')
-            self.env_subs['distro_id'] = distro.id()
-            if self.env_subs['distro_id'] != 'ubuntu':
-                print("WARNING distro id %s not supported by Nvidia " % self.env_subs['distro_id'])
-                sys.exit(1)
-            self.env_subs['distro_version'] = distro.version()
-            if self.env_subs['distro_version'] not in self.supported_versions:
-                print("WARNING distro version %s not in supported list by Nvidia " % self.env_subs['distro_id'])
-                sys.exit(1)
-                # TODO(tfoote) add a standard mechanism for checking preconditions and disabling plugins
-
-        return self.env_subs
+    def get_environment_subs(self, cliargs={}):
+        if not self._env_subs:
+            self._env_subs = {}
+            self._env_subs['user_id'] = os.getuid()
+            self._env_subs['username'] = getpass.getuser()
+            self._env_subs['DISPLAY'] = os.getenv('DISPLAY')
+
+        # non static elements test every time
+        build_detector_image()
+        dist, ver, codename = detect_os(cliargs['base_image'])
+        self._env_subs['image_distro_id'] = dist
+        if self._env_subs['image_distro_id'] not in self.supported_distros:
+            print("WARNING distro id %s not supported by Nvidia supported " % self._env_subs['image_distro_id'], self.supported_distros)
+            sys.exit(1)
+        self._env_subs['image_distro_version'] = ver
+        if self._env_subs['image_distro_version'] not in self.supported_versions:
+            print("WARNING distro version %s not in supported list by Nvidia supported versions" % self._env_subs['image_distro_version'], self.supported_versions)
+            sys.exit(1)
+            # TODO(tfoote) add a standard mechanism for checking preconditions and disabling plugins
+
+        return self._env_subs
 
     def get_preamble(self, cliargs):
         preamble = pkgutil.get_data('rocker', 'templates/%s_preamble.Dockerfile.em' % self.name).decode('utf-8')
-        return em.expand(preamble, self.get_environment_subs())
+        return em.expand(preamble, self.get_environment_subs(cliargs))
 
     def get_snippet(self, cliargs):
         snippet = pkgutil.get_data('rocker', 'templates/%s_snippet.Dockerfile.em' % self.name).decode('utf-8')
-        return em.expand(snippet, self.get_environment_subs())
+        return em.expand(snippet, self.get_environment_subs(cliargs))
 
     def get_docker_args(self, cliargs):
         xauth = self.xauth
@@ -164,17 +170,17 @@ def get_name():
         return 'pulse'
 
     def __init__(self):
-        self.env_subs = None
+        self._env_subs = None
         self.name = PulseAudio.get_name()
 
 
     def get_environment_subs(self):
-        if not self.env_subs:
-            self.env_subs = {}
-            self.env_subs['user_id'] = os.getuid()
-            self.env_subs['XDG_RUNTIME_DIR'] = os.getenv('XDG_RUNTIME_DIR')
-            self.env_subs['audio_group_id'] = grp.getgrnam('audio').gr_gid
-        return self.env_subs
+        if not self._env_subs:
+            self._env_subs = {}
+            self._env_subs['user_id'] = os.getuid()
+            self._env_subs['XDG_RUNTIME_DIR'] = os.getenv('XDG_RUNTIME_DIR')
+            self._env_subs['audio_group_id'] = grp.getgrnam('audio').gr_gid
+        return self._env_subs
 
     def get_preamble(self, cliargs):
         return ''
@@ -204,7 +210,6 @@ def get_name():
         return 'home'
 
     def __init__(self):
-        self.env_subs = None
         self.name = HomeDir.get_name()
 
     def get_docker_args(self, cliargs):
@@ -223,14 +228,14 @@ def get_name():
         return 'user'
 
     def get_environment_subs(self):
-        if not self.env_subs:
-            self.env_subs = {}
-            self.env_subs['user_id'] = os.getuid()
-            self.env_subs['username'] = getpass.getuser()
-        return self.env_subs
+        if not self._env_subs:
+            self._env_subs = {}
+            self._env_subs['user_id'] = os.getuid()
+            self._env_subs['username'] = getpass.getuser()
+        return self._env_subs
 
     def __init__(self):
-        self.env_subs = None
+        self._env_subs = None
         self.name = User.get_name()
 
     def get_snippet(self, cliargs):

src/rocker/templates/nvidia_preamble.Dockerfile.em

@@ -1,2 +1,2 @@
 # Ubuntu 16.04 with nvidia-docker2 beta opengl support
-FROM nvidia/opengl:1.0-glvnd-devel-@(distro_id)@(distro_version) as glvnd
+FROM nvidia/opengl:1.0-glvnd-devel-@(image_distro_id.lower())@(image_distro_version) as glvnd

src/rocker/templates/nvidia_snippet.Dockerfile.em

@@ -1,16 +1,28 @@
+@[if image_distro_version == '16.04']@
 # Open nvidia-docker2 GL support
 COPY --from=glvnd /usr/local/lib/x86_64-linux-gnu /usr/local/lib/x86_64-linux-gnu
 COPY --from=glvnd /usr/local/lib/i386-linux-gnu /usr/local/lib/i386-linux-gnu
 COPY --from=glvnd /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
 COPY --from=glvnd /usr/lib/i386-linux-gnu /usr/lib/i386-linux-gnu
-
-COPY --from=glvnd /usr/local/share/glvnd/egl_vendor.d/10_nvidia.json /usr/local/share/glvnd/egl_vendor.d/10_nvidia.json
-
 # if the path is alreaady present don't fail because of being unable to append
 RUN ( echo '/usr/local/lib/x86_64-linux-gnu' >> /etc/ld.so.conf.d/glvnd.conf && ldconfig || grep -q /usr/local/lib/x86_64-linux-gnu /etc/ld.so.conf.d/glvnd.conf ) && \
     ( echo '/usr/local/lib/i386-linux-gnu' >> /etc/ld.so.conf.d/glvnd.conf &&  ldconfig || grep -q /usr/local/lib/i386-linux-gnu /etc/ld.so.conf.d/glvnd.conf )
-
 ENV LD_LIBRARY_PATH /usr/local/lib/x86_64-linux-gnu:/usr/local/lib/i386-linux-gnu${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
 
+COPY --from=glvnd /usr/local/share/glvnd/egl_vendor.d/10_nvidia.json /usr/local/share/glvnd/egl_vendor.d/10_nvidia.json
+@[else if image_distro_version == '18.04']@
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglvnd0\
+	libgl1 \
+	libglx0 \
+	libegl1 \
+	libgles2 \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=glvnd /usr/share/glvnd/egl_vendor.d/10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json
+@[end if]@
+
+
+
+
 ENV NVIDIA_VISIBLE_DEVICES ${NVIDIA_VISIBLE_DEVICES:-all}
 ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES:+$NVIDIA_DRIVER_CAPABILITIES,}graphics

stdeb.cfg

@@ -1,7 +1,7 @@
 [rocker]
 Debian-Version: 100
 ;Depends: PYTHON 3 ONLY
-Depends3: python3-distro, python3-docker, python3-empy, python3-pexpect
+Depends3: python3-docker, python3-empy, python3-pexpect
 ;Conflicts: python3-rocker
 Conflicts3: python-rocker
 Suite: xenial yakkety zesty artful bionic stretch buster

test/test_nvidia.py

@@ -23,22 +23,30 @@
 
 from io import BytesIO as StringIO
 
-from rocker.cli import DockerImageGenerator, list_plugins
+from rocker.cli import DockerImageGenerator
+from rocker.cli import list_plugins
+from rocker.core import get_docker_client
 
 class NvidiaTest(unittest.TestCase):
     @classmethod
     def setUpClass(self):
-        client = docker.from_env()
-        dockerfile = """
-FROM ubuntu:xenial
+        client = get_docker_client()
+        self.dockerfile_tags = []
+        for distro_version in ['xenial', 'bionic']:
+            dockerfile = """
+FROM ubuntu:%(distro_version)s
 
 RUN apt-get update && apt-get install glmark2 -y && apt-get clean
 
 CMD glmark2 --validate
 """
-        self.dockerfile_tag = 'rocker_test_glmark2'
-        iof = StringIO(dockerfile.encode())
-        im = client.images.build(fileobj = iof, tag=self.dockerfile_tag)
+            dockerfile_tag = 'testfixture_%s_glmark2' % distro_version
+            iof = StringIO((dockerfile % locals()).encode())
+            im = client.build(fileobj = iof, tag=dockerfile_tag)
+            for e in im:
+                pass
+                #print(e)
+            self.dockerfile_tags.append(dockerfile_tag)
 
     def setUp(self):
         # Work around interference between empy Interpreter
@@ -52,14 +60,16 @@ def setUp(self):
         em.Interpreter._wasProxyInstalled = False
 
     def test_no_nvidia_glmark2(self):
-        dig = DockerImageGenerator([], [], self.dockerfile_tag)
-        self.assertEqual(dig.build(), 0)
-        self.assertNotEqual(dig.run(), 0)
+        for tag in self.dockerfile_tags:
+            dig = DockerImageGenerator([], {}, tag)
+            self.assertEqual(dig.build(), 0)
+            self.assertNotEqual(dig.run(), 0)
 
     def test_nvidia_glmark2(self):
         plugins = list_plugins()
         desired_plugins = ['nvidia', 'user']
         active_extensions = [e() for e in plugins.values() if e.get_name() in desired_plugins]
-        dig = DockerImageGenerator(active_extensions, '', self.dockerfile_tag)
-        self.assertEqual(dig.build(), 0)
-        self.assertEqual(dig.run(), 0)
+        for tag in self.dockerfile_tags:
+            dig = DockerImageGenerator(active_extensions, {}, tag)
+            self.assertEqual(dig.build(), 0)
+            self.assertEqual(dig.run(), 0)