Merge branch 'master' into eval_cmd_on_init

.github/scripts/hygiene.sh

@@ -1,36 +1,18 @@
 #!/bin/bash -xue
 
-PR_COMMIT_RANGE=
-if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
-  PR_COMMIT_RANGE="$GITHUB_REF_SHA...$GITHUB_SHA"
-fi
-CI_BRANCH=${GITHUB_REF##*/}
+. .github/scripts/preamble.sh
 
-
-echo "PR_COMMIT_RANGE=$PR_COMMIT_RANGE"
-echo "GITHUB_SHA=$GITHUB_SHA"
-if [[ $GITHUB_EVENT_NAME = 'pull_request' ]] ; then
-  FETCH_HEAD=$(git rev-parse FETCH_HEAD)
-  echo "FETCH_HEAD=$FETCH_HEAD"
-else
-  FETCH_HEAD=$GITHUB_SHA
+if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ "x" = "x$BASE_REF_SHA$PR_REF_SHA" ] ; then
+  echo "Variables BASE_REF_SHA and PR_REF_SHA must be defined in a pull request job"
+  exit 2
 fi
+# Don't use BASE_REF_SHA and PR_REF_SHA on non pull request jobs, they are not
+# defined. See .github/workflows/ci.yml hygiene job.
 
-if [[ $GITHUB_EVENT_NAME = 'push' ]] ; then
-  if ! git cat-file -e "$GITHUB_SHA" 2> /dev/null ; then
-    echo 'GITHUB_SHA does not exist - CI failure'
-    exit 1
-  fi
-else
-  if [[ $GITHUB_SHA != $(git rev-parse FETCH_HEAD) ]] ; then
-    echo 'WARNING! Travis GITHUB_SHA and FETCH_HEAD do not agree!'
-    if git cat-file -e "$GITHUB_SHA" 2> /dev/null ; then
-      echo 'GITHUB_SHA exists, so going with it'
-    else
-      echo 'GITHUB_SHA does not exist; setting to FETCH_HEAD'
-      GITHUB_SHA=$FETCH_HEAD
-    fi
-  fi
+if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+  # needed or git diffs and rev-list
+  git fetch origin master
+  git fetch origin $GITHUB_REF
 fi
 
 CheckConfigure () {
@@ -55,6 +37,8 @@ CheckConfigure () {
       echo -e "[\e[31mERROR\e[0m] configure.ac in $1 doesn't generate configure, \
 please run make configure and fixup the commit"
       ERROR=1
+    else
+      echo "configure ok for $1"
     fi
   fi
 }
@@ -63,22 +47,36 @@ set +x
 
 ERROR=0
 
+###
+# Check configure
+###
+
+(set +x ; echo -en "::group::check configure\r") 2>/dev/null
+case $GITHUB_EVENT_NAME in
+  push)
+    CheckConfigure "$GITHUB_SHA"
+    ;;
+  pull_request)
+    for commit in $(git rev-list $BASE_REF_SHA...$PR_REF_SHA --reverse)
+    do
+      echo "check configure for $commit"
+      CheckConfigure "$commit"
+    done
+    ;;
+  *)
+    echo "no configure to check for unknown event"
+    ;;
+esac
+(set +x ; echo -en "::endgroup::check configure\r") 2>/dev/null
+
+
 ###
 # Check install.sh
 ###
 
 if [ "$GITHUB_EVENT_NAME" = "pull_request" ] ; then
-  CUR_HEAD=$GITHUB_REF_SHA
-  PR_HEAD=$GITHUB_SHA
-  DEEPEN=50
-  while ! git merge-base "$CUR_HEAD" "$PR_HEAD" >& /dev/null
-  do
-    echo "Deepening $CI_BRANCH by $DEEPEN commits"
-    git fetch origin --deepen=$DEEPEN "$CI_BRANCH"
-    ((DEEPEN*=2))
-  done
-  MERGE_BASE=$(git merge-base "$CUR_HEAD" "$PR_HEAD")
-  if ! git diff "$MERGE_BASE..$PR_HEAD" --name-only --exit-code -- shell/install.sh > /dev/null ; then
+  (set +x ; echo -en "::group::check install.sh\r") 2>/dev/null
+  if ! git diff "$BASE_REF_SHA..$PR_REF_SHA" --name-only --exit-code -- shell/install.sh > /dev/null ; then
     echo "shell/install.sh updated - checking it"
     eval $(grep '^\(OPAM_BIN_URL_BASE\|DEV_VERSION\|VERSION\)=' shell/install.sh)
     echo "OPAM_BIN_URL_BASE=$OPAM_BIN_URL_BASE"
@@ -109,28 +107,18 @@ if [ "$GITHUB_EVENT_NAME" = "pull_request" ] ; then
       echo "That can't be right..."
       ERROR=1
     fi
+  else
+    echo "No changes in install.sh"
   fi
+  (set +x ; echo -en "::endgroup::check install.sh\r") 2>/dev/null
 fi
 
-###
-# Check configure
-###
-
-if [[ -z $PR_COMMIT_RANGE ]]
-then CheckConfigure "$GITHUB_SHA"
-else
-  if [[ $GITHUB_EVENT_NAME = 'pull_request' ]]
-  then PR_COMMIT_RANGE=$MERGE_BASE..$GITHUB_SHA
-  fi
-  for commit in $(git rev-list "$PR_COMMIT_RANGE" --reverse)
-  do
-    CheckConfigure "$commit"
-  done
-fi
 
 ###
 # Check src_ext patches
 ###
+
+(set +x ; echo -en "::group::check src_ext patches\r") 2>/dev/null
 # Check that the lib-ext/lib-pkg patches are "simple"
 make -C src_ext PATCH="busybox patch" clone
 make -C src_ext PATCH="busybox patch" clone-pkg
@@ -143,4 +131,6 @@ if [[ $(find patches -name \*.old | wc -l) -ne 0 ]] ; then
   ERROR=1
 fi
 cd ..
+(set +x ; echo -en "::endgroup::check src_ext patches\r") 2>/dev/null
+
 exit $ERROR

.github/workflows/ci.yml

@@ -216,3 +216,23 @@ jobs:
         OCAML_VERSION: ${{ matrix.ocamlv }}
         OPAM_UPGRADE: 1
       run: bash -exu .github/scripts/main.sh
+
+####
+# Around opam tests
+###
+  hygiene:
+    needs: [ archives-cache ]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: src ext archives cache
+      uses: actions/cache@v2
+      with:
+        path: src_ext/archives
+        key: archives-${{ hashFiles('src_ext/Makefile.sources', 'src_ext/Makefile') }}
+    - name: Hygiene
+      env:
+        # Defined only on pull request jobs
+        BASE_REF_SHA: ${{ github.event.pull_request.base.sha }}
+        PR_REF_SHA: ${{ github.event.pull_request.head.sha }}
+      run: bash -exu .github/scripts/hygiene.sh

master_changes.md

@@ -13,7 +13,7 @@ New option/command/subcommand are prefixed with ◈.
 
 ## Init
   * Fix sandbox check with not yet set opam environment variables [#4370 @rjbou - fix #4368]
-  * Sandboxing check: use configured temp dir and cleanup afterwards [#4466 @AltGr]
+  * Sandboxing check: use configured temp dir and cleanup afterwards [#4467 @AltGr]
   * Print shell-appropriate eval command on `opam init` [#4427 @freevoid]
 
 ## Config Upgrade
@@ -22,6 +22,7 @@ New option/command/subcommand are prefixed with ◈.
 ## Install
   * The stdout of `pre-` and `post-session` hooks is now propagated to the user [#4382 @AltGr - fix #4359]
   * `post-install` hooks are allowed to modify or remove installed files, the but not add new ones. Those changes are integrated in changes file [#4388 @lefessan]
+  * ◈ Add `--download-only` flag [#4071 @Armael @rjbou - fix #4036]
 
 ## Remove
   * Fix `opam remove --autoremove <PKG>` to not autoremove unrelated packages [#4369 @AltGr - fix #4250 #4332]
@@ -69,6 +70,7 @@ New option/command/subcommand are prefixed with ◈.
   * Add support for NetBSD and DragonFlyBSD [#4396 @kit-ty-kate]
   * Fix OpenBSD, FreeBSD and Gentoo: Allow short names and full name paths for ports-based systems [#4396 @kit-ty-kate]
   * Handle the case where `os-family=ubuntu` as `os-family=debian` [#4441 @alan-j-hu]
+  * ◈ Provide `opam update --depexts` to request an update of the system package manager databases [#4379 @AltGr - fix #4355]
 
 ## Sandbox
   *
@@ -116,17 +118,17 @@ New option/command/subcommand are prefixed with ◈.
   * Internal caches: use size checks from Marshal [#4430 @AltGr]
   * openssl invocation: Fix permission denied fallback [#4449 @Blaisorblade - fix #4448]
   * Add debug & verbose log for patch & subst application [#4464 @rjbou - fix #4453]
-  * Be more robust w.r.t. new caches updates when `--read-only` is not used [#4466 @AltGr - fix #4354]
+  * Be more robust w.r.t. new caches updates when `--read-only` is not used [#4467 @AltGr - fix #4354]
 
 ## Test
   * Ensure that a cold `dune runtest` works [#4375 @emillon]
   * Use dune "expected" convention for patcher test [#4395 @emillon]
   * Add var/option test [#4428 @rjbou]
-  * patcher: fix local [#4466 @AltGr]
-  * Add github actions [#4463 @rjbou]
-  * Add reftests to github actions [#4466 @rjbou]
-  * Add opam file 1.2 -> 2.0 upgrade test [#4466 @rjbou]
   * Add cli versioning test [#4385 @rjbou]
+  * Add opam file 1.2 -> 2.0 upgrade test [#4467 @rjbou]
+  * patcher: fix local [#4467 @AltGr]
+  * Add github actions [#4463 @rjbou]
+  * Add reftests to github actions [#4467 @rjbou]
 
 ## Shell
   * Update completion scripts with `opam var` instead of `opam config list` [#4428 @rjbou]

src/client/opamClient.ml

@@ -1076,7 +1076,7 @@ let filter_unpinned_locally t atoms f =
     atoms
 
 let install_t t ?ask ?(ignore_conflicts=false) ?(depext_only=false)
-    atoms add_to_roots ~deps_only ~assume_built =
+    ?(download_only=false) atoms add_to_roots ~deps_only ~assume_built =
   log "INSTALL %a" (slog OpamFormula.string_of_atoms) atoms;
   let names = OpamPackage.Name.Set.of_list (List.rev_map fst atoms) in
 
@@ -1229,22 +1229,23 @@ let install_t t ?ask ?(ignore_conflicts=false) ?(depext_only=false)
       in
       let t, res =
         OpamSolution.apply ?ask t ~requested:names ?add_roots
-          ~assume_built solution in
+          ~download_only ~assume_built solution in
       t, Some (Success res)
   in
   OpamStd.Option.iter (OpamSolution.check_solution t) solution;
   t
 
 let install t ?autoupdate ?add_to_roots
     ?(deps_only=false) ?(ignore_conflicts=false) ?(assume_built=false)
-    ?(depext_only=false) names =
+    ?(download_only=false) ?(depext_only=false) names =
   let atoms = OpamSolution.sanitize_atom_list ~permissive:true t names in
   let autoupdate_atoms = match autoupdate with
     | None -> atoms
     | Some a -> OpamSolution.sanitize_atom_list ~permissive:true t a
   in
   let t = update_dev_packages_t autoupdate_atoms t in
-  install_t t ~ignore_conflicts ~depext_only atoms add_to_roots ~deps_only ~assume_built
+  install_t t atoms add_to_roots
+    ~ignore_conflicts ~depext_only ~deps_only ~download_only ~assume_built
 
 let remove_t ?ask ~autoremove ~force atoms t =
   log "REMOVE autoremove:%b %a" autoremove

src/client/opamClient.mli

@@ -51,14 +51,15 @@ val reinit:
 val install:
   rw switch_state ->
   ?autoupdate:atom list -> ?add_to_roots:bool -> ?deps_only:bool ->
-  ?ignore_conflicts:bool -> ?assume_built:bool -> ?depext_only:bool ->
-  atom list ->
+  ?ignore_conflicts:bool -> ?assume_built:bool -> ?download_only:bool ->
+  ?depext_only:bool -> atom list ->
   rw switch_state
 
 (** Low-level version of [reinstall], bypassing the package name sanitization
     and dev package update, and offering more control *)
 val install_t:
-  rw switch_state -> ?ask:bool -> ?ignore_conflicts:bool -> ?depext_only:bool ->
+  rw switch_state ->
+  ?ask:bool -> ?ignore_conflicts:bool -> ?depext_only:bool -> ?download_only:bool ->
   atom list -> bool option -> deps_only:bool -> assume_built:bool ->
   rw switch_state
 

src/client/opamCommands.ml

@@ -1526,6 +1526,10 @@ let install cli =
     mk_flag ~cli (cli_from cli2_1) ["ignore-conflicts"]
       "Used with $(b,--deps-only), ignores conflicts of given package"
   in
+  let download_only =
+    mk_flag ~cli (cli_from cli2_1) ["download-only"]
+      "Fetch the sources of the packages, but don't build or install anything."
+  in
   let restore =
     mk_flag ~cli cli_original ["restore"]
       "Attempt to restore packages that were marked for installation but have \
@@ -1556,7 +1560,7 @@ let install cli =
   let install
       global_options build_options add_to_roots deps_only ignore_conflicts
       restore destdir assume_built check recurse subpath depext_only
-      atoms_or_locals () =
+      download_only atoms_or_locals () =
     apply_global_options global_options;
     apply_build_options build_options;
     if atoms_or_locals = [] && not restore then
@@ -1617,7 +1621,7 @@ let install cli =
     let st =
       OpamClient.install st atoms
         ~autoupdate:pure_atoms ?add_to_roots ~deps_only ~ignore_conflicts
-        ~assume_built ~depext_only
+        ~assume_built ~depext_only ~download_only
     in
     match destdir with
     | None -> `Ok ()
@@ -1630,7 +1634,7 @@ let install cli =
     Term.(const install $global_options cli $build_options cli
           $add_to_roots $deps_only $ignore_conflicts $restore $destdir
           $assume_built cli $check $recurse cli $subpath cli $depext_only
-          $atom_or_local_list)
+          $download_only $atom_or_local_list)
 
 (* REMOVE *)
 let remove_doc = "Remove a list of packages."
@@ -1823,6 +1827,11 @@ let update cli =
     mk_flag ~cli cli_original ["development"]
       "Update development packages (skipping repositories unless \
        $(b,--repositories) is also specified)." in
+  let depexts_only =
+    mk_flag ~cli (cli_from cli2_1) ["depexts"]
+      "Request the system package manager to update its databases (skipping \
+       all opam packages, unless $(b,--development) or $(b,--repositories) is \
+       also specified). This generally requires $(b,sudo) rights." in
   let upgrade =
     mk_flag ~cli cli_original ["u";"upgrade"]
       "Automatically run $(b,opam upgrade) after the update." in
@@ -1842,13 +1851,15 @@ let update cli =
        $(b,--upgrade), applies to the upgrade step: that is $(b,opam update \
        --upgrade --check) behaves like $(b,opam update && opam upgrade --check), \
        returning 0 if there are available upgrades, rather than upstream updates." in
-  let update global_options jobs names repos_only dev_only all
+  let update global_options jobs names repos_only dev_only depexts_only all
       check upgrade () =
     apply_global_options global_options;
     OpamStateConfig.update
       ?jobs:OpamStd.Option.Op.(jobs >>| fun j -> lazy j)
       ();
     OpamClientConfig.update ();
+    if depexts_only then OpamSysInteract.update ();
+    if depexts_only && not (repos_only || dev_only) then () else
     OpamGlobalState.with_ `Lock_write @@ fun gt ->
     let success, changed, rt =
       OpamClient.update gt
@@ -1871,7 +1882,7 @@ let update cli =
   in
   mk_command cli cli_original "update" ~doc ~man
   Term.(const update $global_options cli $jobs_flag cli cli_original $name_list
-        $repos_only $dev_only $all $check $upgrade)
+        $repos_only $dev_only $depexts_only $all $check $upgrade)
 
 (* UPGRADE *)
 let upgrade_doc = "Upgrade the installed package to latest version."
@@ -2717,7 +2728,8 @@ let switch cli =
            if no_action || OpamFormula.satisfies_depends st.installed invariant
            then st
            else OpamClient.install_t
-               st ~ask:true [] None ~deps_only:false ~assume_built:false
+               st ~ask:true [] None
+               ~deps_only:false ~assume_built:false
          in
          OpamSwitchState.drop st;
          `Ok ())