Merge branch 'master' into mikael/rti-openssl

create_jenkins_job.py

@@ -84,7 +84,7 @@ def main(argv=None):
         'use_isolated_default': 'true',
         'colcon_mixin_url': 'https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml',
         'build_args_default': '--event-handlers console_cohesion+ console_package_list+ --cmake-args -DINSTALL_EXAMPLES=OFF -DSECURITY=ON',
-        'test_args_default': "--event-handlers console_direct+ --executor sequential --retest-until-pass 2 --ctest-args -LE xfail --pytest-args -m 'not xfail'",
+        'test_args_default': '--event-handlers console_direct+ --executor sequential --retest-until-pass 2 --ctest-args -LE xfail --pytest-args -m "not xfail"',
         'compile_with_clang_default': 'false',
         'enable_coverage_default': 'false',
         'dont_notify_every_unstable_build': 'false',
@@ -360,8 +360,8 @@ def create_job(os_name, job_name, template_file, additional_dict):
             job_name = job_name[:15]
         test_args_default = os_configs.get(os_name, data).get('test_args_default', data['test_args_default'])
         test_args_default = test_args_default.replace('--retest-until-pass', '--retest-until-fail')
-        test_args_default = test_args_default.replace('--ctest-args -LE xfail', "--ctest-args -LE '(linter|xfail)'")
-        test_args_default = test_args_default.replace("--pytest-args -m 'not xfail'", "--pytest-args -m 'not linter and not xfail'")
+        test_args_default = test_args_default.replace('--ctest-args -LE xfail', '--ctest-args -LE "(linter|xfail)"')
+        test_args_default = test_args_default.replace('--pytest-args -m "not xfail"', '--pytest-args -m "not linter and not xfail"')
         if job_os_name == 'linux-aarch64':
             # skipping known to be flaky tests https://github.com/ros2/rviz/issues/368
             test_args_default += ' --packages-skip rviz_common rviz_default_plugins rviz_rendering rviz_rendering_tests'
@@ -374,8 +374,9 @@ def create_job(os_name, job_name, template_file, additional_dict):
 
         # configure nightly triggered job for excluded test
         job_name = 'nightly_' + job_os_name + '_xfail'
-        test_args_default = data['test_args_default'].replace('--ctest-args -LE xfail', '--ctest-args -L xfail')
-        test_args_default = test_args_default.replace("--pytest-args -m 'not xfail'", '--pytest-args -m xfail --runxfail')
+        test_args_default = os_configs.get(os_name, data).get('test_args_default', data['test_args_default'])
+        test_args_default = test_args_default.replace('--ctest-args -LE xfail', '--ctest-args -L xfail')
+        test_args_default = test_args_default.replace('--pytest-args -m "not xfail"', '--pytest-args -m xfail --runxfail')
         create_job(os_name, job_name, 'ci_job.xml.em', {
             'cmake_build_type': 'None',
             'time_trigger_spec': PERIODIC_JOB_SPEC,

job_templates/ci_job.xml.em

@@ -341,6 +341,8 @@ if "!CI_BUILD_ARGS!" NEQ "" (
   set "CI_ARGS=!CI_ARGS! --build-args !CI_BUILD_ARGS!"
 )
 if "!CI_TEST_ARGS!" NEQ "" (
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:"=\"!"
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:|=^|!"
   set "CI_ARGS=!CI_ARGS! --test-args !CI_TEST_ARGS!"
 )
 echo Using args: !CI_ARGS!
@@ -364,7 +366,7 @@ rem "Finding the ReleaseId is much easier with powershell than cmd"
 powershell $(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion').ReleaseId > release_id.txt
 set /p RELEASE_ID=< release_id.txt
 set BUILD_ARGS=--build-arg WINDOWS_RELEASE_ID=%RELEASE_ID%
-docker build  %BUILD_ARGS% -t %CONTAINER_NAME% -f %DOCKERFILE% windows_docker_resources || exit /b %ERRORLEVEL%
+docker build  %BUILD_ARGS% -t %CONTAINER_NAME% -f %DOCKERFILE% windows_docker_resources || exit /b !ERRORLEVEL!
 echo "# END SECTION"
 
 echo "# BEGIN SECTION: Determine arguments"
@@ -431,6 +433,8 @@ if "!CI_BUILD_ARGS!" NEQ "" (
   set "CI_ARGS=!CI_ARGS! --build-args !CI_BUILD_ARGS!"
 )
 if "!CI_TEST_ARGS!" NEQ "" (
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:"=\"!"
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:|=^|!"
   set "CI_ARGS=!CI_ARGS! --test-args !CI_TEST_ARGS!"
 )
 echo Using args: !CI_ARGS!
@@ -442,8 +446,8 @@ powershell -Command "if ($(docker ps -q) -ne $null) { docker stop $(docker ps -q
 
 rem If isolated_network doesn't already exist, create it
 set NETWORK_NAME=isolated_network
-docker network inspect %NETWORK_NAME% 2>nul 1>nul || docker network create -d nat -o com.docker.network.bridge.enable_icc=false %NETWORK_NAME%  || exit /b %ERRORLEVEL%
-docker run --isolation=process --rm --net=%NETWORK_NAME% -e ROS_DOMAIN_ID=1 -e CI_ARGS="%CI_ARGS%" -v "%cd%":"C:\ci" %CONTAINER_NAME%  || exit /b %ERRORLEVEL%
+docker network inspect %NETWORK_NAME% 2>nul 1>nul || docker network create -d nat -o com.docker.network.bridge.enable_icc=false %NETWORK_NAME%  || exit /b !ERRORLEVEL!
+docker run --isolation=process --rm --net=%NETWORK_NAME% -e ROS_DOMAIN_ID=1 -e CI_ARGS="%CI_ARGS%" -v "%cd%":"C:\ci" %CONTAINER_NAME%  || exit /b !ERRORLEVEL!
 echo "# END SECTION"
 @[else]@
 @{ assert False, 'Unknown os_name: ' + os_name }@

job_templates/packaging_job.xml.em

@@ -34,6 +34,7 @@
     cmake_build_type=cmake_build_type,
     build_args_default=build_args_default,
     os_name=os_name,
+    use_isolated_default=False,
 ))@
 @(SNIPPET(
     'property_parameter-definition_rmw_implementations',
@@ -118,6 +119,7 @@ use_cyclonedds: ${build.buildVariableResolver.resolve('CI_USE_CYCLONEDDS')}, <br
 use_fastrtps_static: ${build.buildVariableResolver.resolve('CI_USE_FASTRTPS_STATIC')}, <br/>
 use_fastrtps_dynamic: ${build.buildVariableResolver.resolve('CI_USE_FASTRTPS_DYNAMIC')}, <br/>
 use_opensplice: ${build.buildVariableResolver.resolve('CI_USE_OPENSPLICE')}, <br/>
+isolated: ${build.buildVariableResolver.resolve('CI_ISOLATED')}, <br/>
 colcon_mixin_url: ${build.buildVariableResolver.resolve('CI_COLCON_MIXIN_URL')}, <br/>
 cmake_build_type: ${build.buildVariableResolver.resolve('CI_CMAKE_BUILD_TYPE')}, <br/>
 build_args: ${build.buildVariableResolver.resolve('CI_BUILD_ARGS')}, <br/>
@@ -175,6 +177,9 @@ fi
 if [ -n "${CI_MIXED_ROS_OVERLAY_PKGS+x}" ]; then
   export CI_ARGS="$CI_ARGS --mixed-ros-overlay-pkgs $CI_MIXED_ROS_OVERLAY_PKGS"
 fi
+if [ "$CI_ISOLATED" = "true" ]; then
+  export CI_ARGS="$CI_ARGS --isolated"
+fi
 if [ "${CI_UBUNTU_DISTRO}" = "focal" ]; then
   export CI_ROS1_DISTRO=noetic
 elif [ "${CI_UBUNTU_DISTRO}" = "bionic" ]; then
@@ -303,6 +308,9 @@ set "CI_ARGS=!CI_ARGS! --repo-file-url !CI_ROS2_REPOS_URL!"
 if "!CI_TEST_BRIDGE!" == "true" (
   set "CI_ARGS=!CI_ARGS! --test-bridge"
 )
+if "!CI_ISOLATED!" == "true" (
+  set "CI_ARGS=!CI_ARGS! --isolated"
+)
 if "!CI_COLCON_MIXIN_URL!" NEQ "" (
   set "CI_ARGS=!CI_ARGS! --colcon-mixin-url !CI_COLCON_MIXIN_URL!"
 )
@@ -319,6 +327,8 @@ if "!CI_BUILD_ARGS!" NEQ "" (
   set "CI_ARGS=!CI_ARGS! --build-args !CI_BUILD_ARGS!"
 )
 if "!CI_TEST_ARGS!" NEQ "" (
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:"=\"!"
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:|=^|!"
   set "CI_ARGS=!CI_ARGS! --test-args !CI_TEST_ARGS!"
 )
 echo Using args: !CI_ARGS!
@@ -342,7 +352,7 @@ rem "Finding the ReleaseId is much easier with powershell than cmd"
 powershell $(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion').ReleaseId > release_id.txt
 set /p RELEASE_ID=&lt; release_id.txt
 set BUILD_ARGS=--build-arg WINDOWS_RELEASE_ID=%RELEASE_ID%
-docker build  %BUILD_ARGS% -t %CONTAINER_NAME% -f %DOCKERFILE% windows_docker_resources  || exit /b %ERRORLEVEL%
+docker build  %BUILD_ARGS% -t %CONTAINER_NAME% -f %DOCKERFILE% windows_docker_resources  || exit /b !ERRORLEVEL!
 echo "# END SECTION"
 
 echo "# BEGIN SECTION: Determine arguments"
@@ -383,6 +393,9 @@ set "CI_ARGS=!CI_ARGS! --repo-file-url !CI_ROS2_REPOS_URL!"
 if "!CI_TEST_BRIDGE!" == "true" (
   set "CI_ARGS=!CI_ARGS! --test-bridge"
 )
+if "!CI_ISOLATED!" == "true" (
+  set "CI_ARGS=!CI_ARGS! --isolated"
+)
 if "!CI_COLCON_MIXIN_URL!" NEQ "" (
   set "CI_ARGS=!CI_ARGS! --colcon-mixin-url !CI_COLCON_MIXIN_URL!"
 )
@@ -397,6 +410,8 @@ if "!CI_BUILD_ARGS!" NEQ "" (
   set "CI_ARGS=!CI_ARGS! --build-args !CI_BUILD_ARGS!"
 )
 if "!CI_TEST_ARGS!" NEQ "" (
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:"=\"!"
+  set "CI_TEST_ARGS=!CI_TEST_ARGS:|=^|!"
   set "CI_ARGS=!CI_ARGS! --test-args !CI_TEST_ARGS!"
 )
 echo Using args: !CI_ARGS!
@@ -408,8 +423,8 @@ powershell -Command "if ($(docker ps -q) -ne $null) { docker stop $(docker ps -q
 
 rem If isolated_network doesn't already exist, create it
 set NETWORK_NAME=isolated_network
-docker network inspect %NETWORK_NAME% 2>nul 1>nul || docker network create -d nat -o com.docker.network.bridge.enable_icc=false %NETWORK_NAME%  || exit /b %ERRORLEVEL%
-docker run --isolation=process --rm --net=%NETWORK_NAME% -e ROS_DOMAIN_ID=1 -e CI_ARGS="%CI_ARGS%" -v "%cd%":"C:\ci" %CONTAINER_NAME%  || exit /b %ERRORLEVEL%
+docker network inspect %NETWORK_NAME% 2>nul 1>nul || docker network create -d nat -o com.docker.network.bridge.enable_icc=false %NETWORK_NAME%  || exit /b !ERRORLEVEL!
+docker run --isolation=process --rm --net=%NETWORK_NAME% -e ROS_DOMAIN_ID=1 -e CI_ARGS="%CI_ARGS%" -v "%cd%":"C:\ci" %CONTAINER_NAME%  || exit /b !ERRORLEVEL!
 echo "# END SECTION"
 @[else]@
 @{ assert False, 'Unknown os_name: ' + os_name }@

job_templates/snippet/property_parameter-definition.xml.em

@@ -10,6 +10,7 @@
     cmake_build_type=cmake_build_type,
     build_args_default=build_args_default,
     os_name=os_name,
+    use_isolated_default=use_isolated_default,
 ))@
         <hudson.model.BooleanParameterDefinition>
           <name>CI_USE_WHITESPACE_IN_PATHS</name>
@@ -23,11 +24,6 @@ This tests the robustness to whitespace being within the different paths.</descr
     ignore_rmw_default=ignore_rmw_default,
     use_connext_debs_default=use_connext_debs_default,
 ))@
-        <hudson.model.BooleanParameterDefinition>
-          <name>CI_ISOLATED</name>
-          <description>By setting this to True, the build will use the --isolated option.</description>
-          <defaultValue>@(use_isolated_default)</defaultValue>
-        </hudson.model.BooleanParameterDefinition>
         <hudson.model.BooleanParameterDefinition>
           <name>CI_COMPILE_WITH_CLANG</name>
           <description>By setting this to true, the build will run with clang compiler on Linux (currently ignored on non-Linux).</description>

job_templates/snippet/property_parameter-definition_common.xml.em

@@ -93,3 +93,8 @@ choices.remove(cmake_build_type)
           </choices>
         </hudson.model.ChoiceParameterDefinition>
 @[end if]@
+        <hudson.model.BooleanParameterDefinition>
+          <name>CI_ISOLATED</name>
+          <description>By setting this to True, the build will use the --isolated option.</description>
+          <defaultValue>@(use_isolated_default)</defaultValue>
+        </hudson.model.BooleanParameterDefinition>

linux_docker_resources/Dockerfile-CentOS

@@ -141,4 +141,4 @@ RUN chmod 755 /entry_point.sh
 
 ENTRYPOINT ["/entry_point.sh"]
 
-CMD ["matchbox-window-manager > /dev/null 2>&1 & scl enable devtoolset-8 -- python3.6 -u run_ros2_batch.py $CI_ARGS"]
+CMD ["matchbox-window-manager > /dev/null 2>&1 & . /opt/rh/devtoolset-8/enable && python3.6 -u run_ros2_batch.py $CI_ARGS"]

ros2_batch_job/__main__.py

@@ -459,7 +459,7 @@ def run(args, build_function, blacklisted_package_names=None):
         args.os = 'linux'
         from .linux_batch import LinuxBatchJob
         job = LinuxBatchJob(args)
-    elif args.os == 'osx' or platform_name.startswith('darwin'):
+    elif args.os == 'osx' or platform_name.startswith('darwin') or platform_name.startswith('macos'):
         args.os = 'osx'
         from .osx_batch import OSXBatchJob
         job = OSXBatchJob(args)

ros2_batch_job/packaging.py

@@ -136,18 +136,32 @@ def build_and_test_and_package(args, job):
     # Only on Linux and OSX Python scripts have a shebang line
     if args.os in ['linux', 'osx']:
         print('# BEGIN SUBSECTION: rewrite shebang lines')
-        paths_to_files = []
-
-        bin_path = os.path.join(args.installspace, 'bin')
-        bin_contents = [os.path.join(bin_path, name) for name in os.listdir(bin_path)]
-        paths_to_files.extend([path for path in bin_contents if os.path.isfile(path)])
-
-        # Demo nodes are installed to 'lib/<package_name>'
-        lib_path = os.path.join(args.installspace, 'lib')
-        for lib_sub_dir in next(os.walk(lib_path))[1]:
-            sub_dir_path = os.path.join(lib_path, lib_sub_dir)
-            sub_dir_contents = [os.path.join(sub_dir_path, name) for name in os.listdir(sub_dir_path)]
-            paths_to_files.extend([path for path in sub_dir_contents if os.path.isfile(path)])
+
+        def _get_files_from_install(install_prefix):
+            paths_to_files = []
+
+            bin_path = os.path.join(install_prefix, 'bin')
+            if os.path.exists(bin_path):
+                bin_contents = [os.path.join(bin_path, name) for name in os.listdir(bin_path)]
+                paths_to_files.extend([path for path in bin_contents if os.path.isfile(path)])
+
+            # Demo nodes are installed to 'lib/<package_name>'
+            lib_path = os.path.join(install_prefix, 'lib')
+            if os.path.exists(lib_path):
+                for lib_sub_dir in next(os.walk(lib_path))[1]:
+                    sub_dir_path = os.path.join(lib_path, lib_sub_dir)
+                    sub_dir_contents = [os.path.join(sub_dir_path, name) for name in os.listdir(sub_dir_path)]
+                    paths_to_files.extend([path for path in sub_dir_contents if os.path.isfile(path)])
+
+            return paths_to_files
+
+        if not args.isolated:
+            paths_to_files = _get_files_from_install(args.installspace)
+        else:
+            paths_to_files = []
+            for pkg_name in [p for p in os.listdir(args.installspace) if os.path.isdir(p)]:
+                paths_to_files += _get_files_from_install(
+                    os.path.join(args.installspace, pkg_name))
 
         for path in paths_to_files:
             with open(path, 'rb') as h:

ros2_batch_job/windows_batch/__init__.py

@@ -91,7 +91,7 @@ def with_vendors(cmd, **kwargs):
             # Use the env file to call the commands
             # ensure that quoted arguments are passed through as quoted arguments
             cmd = ['env.bat'] + [
-                '"%s"' % c if ' ' in c and not (c.startswith('"') and c.endswith('"')) else c
+                '"%s"' % c if (' ' in c or '|' in c) and not (c.startswith('"') and c.endswith('"')) else c
                 for c in cmd]
             # Pass along to the original runner
             return current_run(cmd, **kwargs)