ci: Sync with ostree

.papr.yml

@@ -18,13 +18,11 @@ cluster:
   container:
     image: registry.fedoraproject.org/fedora:26
 
-# https://bugzilla.redhat.com/show_bug.cgi?id=1483553
-#packages:
-#  - git
-#  - rsync
-
 env:
   HOSTS: vmcheck1 vmcheck2 vmcheck3
+  # TODO use -fsanitize=address
+  CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2'
+  ASAN_OPTIONS: 'detect_leaks=0'  # Right now we're not fully clean, but this gets us use-after-free etc
   CI_PKGS: rsync
 
 tests:
@@ -37,6 +35,7 @@ timeout: 60m
 
 artifacts:
   - test-suite.log
+  - config.log
   - vmcheck
 
 ---
@@ -59,6 +58,11 @@ cluster:
     # FIXME remove this version binding when rpm-md repos are updated
     image: registry.centos.org/centos/centos:7.3.1611
 
+# We only want the sanitizers on Fedora
+env:
+    HOSTS: vmcheck1 vmcheck2 vmcheck3
+    CFLAGS: ''
+
 extra-repos:
   - name: atomic-centos-continuous
     baseurl: https://ci.centos.org/artifacts/sig-atomic/rdgo/centos-continuous/build

Makefile-tests.am

@@ -15,7 +15,7 @@ AM_TESTS_ENVIRONMENT += \
 	GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd)$${GI_TYPELIB_PATH:+:$$GI_TYPELIB_PATH} \
 	$(NULL)
 if BUILDOPT_ASAN
-AM_TESTS_ENVIRONMENT += ASAN_OPTIONS=detect_leaks=false
+AM_TESTS_ENVIRONMENT += BUILDOPT_ASAN=yes ASAN_OPTIONS=detect_leaks=false
 endif
 
 testbin_cppflags = $(AM_CPPFLAGS) -I $(srcdir)/src/libpriv -I $(srcdir)/libglnx -I $(srcdir)/tests/common

ci/build-check.sh

@@ -10,16 +10,3 @@ ${dn}/build.sh
 # support parallel runs right now
 /usr/bin/make check
 make install
-
-# And now a clang build to find unused variables, but only run on Fedora because
-# the CentOS version is ancient anyway and doesn't support all the flags that
-# might get passed to it.
-id=$(. /etc/os-release && echo $ID)
-if [ "$id" == fedora ]; then
-    git clean -dfx
-    export CC=clang
-    export CFLAGS='-Werror=unused-variable -Werror=maybe-uninitialized'
-    build_default
-    # don't actually run the tests, just compile them
-    /usr/bin/make check TESTS=
-fi

ci/build.sh

@@ -18,12 +18,9 @@ elif [ "$id" == centos ]; then
 fi
 
 pkg_upgrade
-
-install_builddeps rpm-ostree
-
-yum install -y /usr/bin/g-ir-scanner # Accidentally omitted
+pkg_install_builddeps rpm-ostree
 # Mostly dependencies for tests
-yum install -y ostree{,-devel,-grub2} createrepo_c /usr/bin/jq PyYAML clang \
+pkg_install ostree{,-devel,-grub2} createrepo_c /usr/bin/jq PyYAML clang \
     libubsan libasan libtsan elfutils fuse sudo python-gobject-base
 
 if [ -n "${CI_PKGS:-}" ]; then
@@ -33,5 +30,5 @@ fi
 # create an unprivileged user for testing
 adduser testuser
 
-rpm -q ostree{,-devel,-grub2}
+export LSAN_OPTIONS=verbosity=1:log_threads=1
 build --enable-installed-tests --enable-gtk-doc

ci/libbuild.sh

@@ -2,51 +2,68 @@
 
 pkg_upgrade() {
     # https://bugzilla.redhat.com/show_bug.cgi?id=1483553
-    if ! yum -y upgrade 2>err.txt; then
-        ecode=$?
-        if grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then
-            rpm --rebuilddb
-            yum -y upgrade
-        else
+    ecode=0
+    yum -y distro-sync 2>err.txt || ecode=$?
+    if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then
+        rpm --rebuilddb
+        yum -y distro-sync
+    else
+        if test ${ecode} '!=' 0; then
             cat err.txt
             exit ${ecode}
         fi
     fi
 }
 
-pkg_install() {
-  yum -y install "$@"
-}
-
 make() {
     /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@"
 }
 
 build() {
     env NOCONFIGURE=1 ./autogen.sh
     ./configure --prefix=/usr --libdir=/usr/lib64 "$@"
-    make
+    make V=1
 }
 
-build_default() {
-    export CFLAGS="${CFLAGS:-} -fsanitize=undefined"
-    build
+pkg_install() {
+    yum -y install "$@"
 }
 
-install_builddeps() {
-    pkg=$1
+pkg_install_if_os() {
+    os=$1
+    shift
+    (. /etc/os-release;
+         if test "${os}" = "${ID}"; then
+            pkg_install "$@"
+         else
+             echo "Skipping installation on OS ${ID}: $@"
+         fi
+    )
+}
 
-    if [ -x /usr/bin/dnf ]; then
-        dnf -y install dnf-plugins-core
-        dnf install -y @buildsys-build
-        dnf install -y 'dnf-command(builddep)'
-        dnf builddep -y $pkg
+pkg_builddep() {
+    # This is sadly the only case where it's a different command
+    if test -x /usr/bin/dnf; then
+        dnf builddep -y "$@"
     else
-        yum install -y make rpm-build
-        yum-builddep -y rpm-ostree
+        yum-builddep -y "$@"
     fi
+}
 
+pkg_install_builddeps() {
+    pkg=$1
+    if test -x /usr/bin/dnf; then
+        yum -y install dnf-plugins-core
+        yum install -y 'dnf-command(builddep)'
+        # Base buildroot
+        pkg_install @buildsys-build
+    else
+        yum -y install yum-utils
+        # Base buildroot, copied from the mock config sadly
+        yum -y install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz
+    fi
     # builddeps+runtime deps
-    yum install -y $pkg
+    pkg_builddep $pkg
+    pkg_install $pkg
     rpm -e $pkg
 }

tests/check/test-basic.sh

@@ -202,22 +202,26 @@ fi
 assert_file_has_content err.txt 'Unknown.*command'
 echo "ok error on unknown command"
 
-cat >test-rpmostree-gi-arch <<EOF
+if ! skip_one_with_asan; then
+    cat >test-rpmostree-gi-arch <<EOF
 #!/usr/bin/python2
 import gi
 gi.require_version("RpmOstree", "1.0")
 from gi.repository import RpmOstree
 assert RpmOstree.get_basearch() == 'x86_64'
 assert RpmOstree.varsubst_basearch('http://example.com/foo/\${basearch}/bar') == 'http://example.com/foo/x86_64/bar'
 EOF
-chmod a+x test-rpmostree-gi-arch
-case $(arch) in
-    x86_64) ./test-rpmostree-gi-arch;;
-    *) echo "Skipping RPM architecture test on $(arch)"
-esac
-echo "ok rpmostree arch"
-
-cat >test-rpmostree-gi <<EOF
+    chmod a+x test-rpmostree-gi-arch
+    case $(arch) in
+        x86_64) ./test-rpmostree-gi-arch
+                echo "ok rpmostree arch"
+                ;;
+        *) echo "ok # SKIP Skipping RPM architecture test on $(arch)"
+    esac
+fi
+
+if ! skip_one_with_asan; then
+    cat >test-rpmostree-gi <<EOF
 #!/usr/bin/python2
 import gi
 gi.require_version("RpmOstree", "1.0")
@@ -226,6 +230,7 @@ assert RpmOstree.check_version(2017, 6)
 # If this fails for you, please come back in a time machine and say hi
 assert not RpmOstree.check_version(3000, 1)
 EOF
-chmod a+x test-rpmostree-gi
-./test-rpmostree-gi
-echo "ok rpmostree version"
+    chmod a+x test-rpmostree-gi
+    ./test-rpmostree-gi
+    echo "ok rpmostree version"
+fi

tests/common/libtest.sh

@@ -342,6 +342,19 @@ ensure_dbus ()
     fi
 }
 
+# https://github.com/ostreedev/ostree/commit/47b4dd1b38e422254afa67756873957c25aeab6d
+# Unfortunately, introspection uses dlopen(), which doesn't quite
+# work when the DSO is compiled with ASAN but the outer executable
+# isn't.
+skip_one_with_asan () {
+    if test -n "${BUILDOPT_ASAN:-}"; then
+        echo "ok # SKIP - built with ASAN"
+        return 0
+    else
+        return 1
+    fi
+}
+
 assert_status_file_jq() {
     status_file=$1; shift
     for expression in "$@"; do