Disable shallow clone by default except for opam repositories

master_changes.md

@@ -33,6 +33,7 @@ users)
   * Make fetching an archive from cache add missing symlinks [#6068 @kit-ty-kate - fix #6064]
   * [BUG] Fix `opam install --deps-only` set direct dependencies as root packages [#6125 @rjbou]
   * [BUG] Fix `opam install --check pkg` when pkg depends on a non-existing package [#6121 @kit-ty-kate]
+  * Disable shallow clone by default except for opam repositories [#6146 @kit-ty-kate - fix #6145]
 
 ## Build (package)
   * ◈ Add `--verbose-on` option to enable verbose mode on specified package names [#5682 @desumn @rjbou]
@@ -155,6 +156,7 @@ users)
   * Add reinstall test for delayed removal of packages [#6139 @rjbou]
   * Add a test showing the behaviour of `opam list --latests-only` [#5375 @kit-ty-kate]
   * Add a test filtering mechanism [#6105 @Keryan-dev]
+  * Add a package fetching test [#6146 @rjbou]
 
 ### Engine
   * Add a test filtering mechanism [#6105 @Keryan-dev]
@@ -194,6 +196,7 @@ users)
 
 ## opam-repository
  * `OpamRepository.fetch_from_cache`: when an archive is found, add a symlink (or copy) for the ones found in opam file but not in cache [#6068 @kit-ty-kate]
+ * `?full_fetch` is now `true` by default instead of `false` [#6146 @kit-ty-kate - fix #6145]
 
 ## opam-state
  * `OpamStateConfig.opamroot_with_provenance`: restore previous behaviour to `OpamStateConfig.opamroot` for compatibility with third party code [#6047 @dra27]

src/repository/opamGit.ml

@@ -62,7 +62,7 @@ module VCS : OpamVCS.VCS = struct
     | Some h -> "refs/remotes/opam-ref-"^h
     | None -> "refs/remotes/opam-ref"
 
-  let fetch ?(full_fetch = false) ?cache_dir ?subpath repo_root repo_url =
+  let fetch ?(full_fetch = true) ?cache_dir ?subpath repo_root repo_url =
     (match subpath with
      | Some sp ->
        git repo_root [ "config"; "--local"; "core.sparseCheckout"; "true" ]

src/repository/opamRepository.mli

@@ -43,8 +43,8 @@ val pull_shared_tree:
   url list -> string download OpamProcess.job
 
 (* Same as [pull_shared_tree], but for a unique label/dirname.
-   If [full_fetch] is true, VCS repository is retrieved with full history (by
-   default, no history). *)
+   If [full_fetch] is set to false, VCS repository is retrieved with shallow
+   history (by default, full history). *)
 val pull_tree:
   string -> ?full_fetch:bool -> ?cache_dir:dirname -> ?cache_urls:url list ->
   ?working_dir:bool -> ?subpath:subpath ->

src/repository/opamRepositoryBackend.mli

@@ -45,8 +45,8 @@ module type S = sig
       [checksum] can be used for retrieval but is NOT checked by this
       function.
 
-      If [full_fetch] is set to true, VCS repository is retrieved with full
-      history (by default, no history).
+      If [full_fetch] is set to false, VCS repository is retrieved with shallow
+      history (by default, full history).
       If [cache_dir] is given, the directory is used by VCS tool as a its cache
       directory.
       If [subpath] is given, only that [subpath] of the url is retrieved. *)

src/repository/opamVCS.mli

@@ -29,8 +29,8 @@ module type VCS = sig
       Be aware that the remote URL might have been changed, so make sure
       to update accordingly.
 
-      If [full_fetch] is set to true, VCS repository is retrieved with full
-      history (by default, no history).
+      If [full_fetch] is set to false, VCS repository is retrieved with shallow
+      history (by default, full history).
       If [cache_dir] is given, the directory is used by VCS tool as a its cache
       directory.
       If [subpath] is given, only that [subpath] of the url is retrieved. *)

tests/reftests/download.test

@@ -238,7 +238,7 @@ Processing  1/1: [bar.1: git]
 + git "config" "--local" "color.ui" "false" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
 + git "remote" "add" "origin" "file://${BASEDIR}/bar" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
 + git "remote" "set-url" "origin" "file://${BASEDIR}/bar" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
-+ git "fetch" "-q" "file://${BASEDIR}/bar" "--update-shallow" "--depth=1" "+HEAD:refs/remotes/opam-ref" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
++ git "fetch" "-q" "file://${BASEDIR}/bar" "--update-shallow" "+HEAD:refs/remotes/opam-ref" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
 + git "reset" "--hard" "refs/remotes/opam-ref" "--" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
 + git "clean" "-fdx" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/bar.1)
 SYSTEM                          rmdir ${OPAMTMP}
@@ -297,7 +297,7 @@ Processing  1/1: [qux.1: git]
 SYSTEM                          mkdir ${BASEDIR}/OPAM/download-cache/git
 + git "init" "--bare" (CWD=${BASEDIR}/OPAM/download-cache/git)
 + git "remote" "set-url" "origin" "https://github.com/ocaml-opam/opam-depext" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1)
-+ git "fetch" "-q" "https://github.com/ocaml-opam/opam-depext" "--update-shallow" "--depth=1" "+HEAD:refs/remotes/opam-ref" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1)
++ git "fetch" "-q" "https://github.com/ocaml-opam/opam-depext" "--update-shallow" "+HEAD:refs/remotes/opam-ref" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1)
 + git "push" git "+refs/remotes/opam-ref:refs/remotes/cd8336413a06dcd0c48d3f48df5d1940" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1)
 + git "reset" "--hard" "refs/remotes/opam-ref" "--" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1)
 + git "clean" "-fdx" (CWD=${BASEDIR}/OPAM/download/.opam-switch/sources/qux.1)

tests/reftests/dune.inc

@@ -776,6 +776,27 @@
    %{targets}
    (run ./run.exe %{exe:../../src/client/opamMain.exe.exe} %{dep:extrasource.test} %{read-lines:testing-env}))))
 
+(rule
+ (alias reftest-fetch-package)
+ (enabled_if (and  (or (<> %{env:TESTALL=1} 0) (= %{env:TESTN0REP0=0} 1))))
+ (action
+  (diff fetch-package.test fetch-package.out)))
+
+(alias
+ (name reftest)
+ (enabled_if (and  (or (<> %{env:TESTALL=1} 0) (= %{env:TESTN0REP0=0} 1))))
+ (deps (alias reftest-fetch-package)))
+
+(rule
+ (targets fetch-package.out)
+ (deps root-N0REP0)
+ (enabled_if (and  (or (<> %{env:TESTALL=1} 0) (= %{env:TESTN0REP0=0} 1))))
+ (package opam)
+ (action
+  (with-stdout-to
+   %{targets}
+   (run ./run.exe %{exe:../../src/client/opamMain.exe.exe} %{dep:fetch-package.test} %{read-lines:testing-env}))))
+
 (rule
  (alias reftest-filter-operators)
  (enabled_if (and  (or (<> %{env:TESTALL=1} 0) (= %{env:TESTN0REP0=0} 1))))

tests/reftests/fetch-package.test

@@ -0,0 +1,146 @@
+N0REP0
+### <a-dev/root.ml>
+print_endline "i'm root file"
+### <pin:a-dev/opam>
+opam-version: "2.0"
+build: [
+  [ "sh" "-c" "git ls-files > files || ls > files" ]
+  [ "sh" "-c" "git rev-list --all --count > hist || echo > hist" ]
+]
+install: [
+  [ "cp" "files" "%{lib}%/%{name}%-files" ]
+  [ "cp" "hist" "%{lib}%/%{name}%-hist" ]
+]
+### tar czf arch.tgz a-dev
+### git -C ./a-dev init -q --initial-branch=master
+### git -C ./a-dev config core.autocrlf false
+### git -C ./a-dev add opam
+### git -C ./a-dev commit -qm "init"
+### git -C ./a-dev add root.ml
+### git -C ./a-dev commit -qm "i'm empty"
+### <pkg:foo-arch.1>
+opam-version: "2.0"
+build: [ "sh" "-c" "ls > files" ]
+install: [ "cp" "files" "%{lib}%/%{name}%-files" ]
+### <pkg:foo-git.1>
+opam-version: "2.0"
+build: [
+  [ "sh" "-c" "git ls-files > files" ]
+  [ "sh" "-c" "git rev-list --all --count > hist" ]
+]
+install: [
+  [ "cp" "files" "%{lib}%/%{name}%-files" ]
+  [ "cp" "hist" "%{lib}%/%{name}%-hist" ]
+]
+### <mkurl.sh>
+p=foo-arch.1
+arch=arch
+file="REPO/packages/${p%.*}/$p/opam"
+basedir=`echo "$BASEDIR" | sed "s/\\\\\\\\/\\\\\\\\\\\\\\\\/g"`
+MD5=$(openssl md5 "$arch.tgz" | cut -d' ' -f2)
+cat << EOF >> "$file"
+url {
+  src: "$arch.tgz"
+  checksum: "md5=$MD5"
+}
+dev-repo: "git+file://${basedir}/a-dev"
+EOF
+
+p=foo-git.1
+file="REPO/packages/${p%.*}/$p/opam"
+cat << EOF >> "$file"
+url {
+  src: "git+file://${basedir}/a-dev"
+}
+dev-repo: "git+file://${basedir}/a-dev"
+EOF
+### sh mkurl.sh
+### opam update
+
+<><> Updating package repositories ><><><><><><><><><><><><><><><><><><><><><><>
+[default] synchronised from file://${BASEDIR}/REPO
+Now run 'opam upgrade' to apply any package updates.
+### opam switch create downloads-repo --empty
+### opam install foo-arch
+The following actions will be performed:
+=== install 1 package
+  - install foo-arch 1
+
+<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+-> retrieved foo-arch.1  (file://${BASEDIR}/arch.tgz)
+-> installed foo-arch.1
+Done.
+### cat OPAM/downloads-repo/lib/foo-arch-files | sort
+files
+opam
+root.ml
+### opam install foo-git
+The following actions will be performed:
+=== install 1 package
+  - install foo-git 1
+
+<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+-> retrieved foo-git.1  (git+file://${BASEDIR}/a-dev)
+-> installed foo-git.1
+Done.
+### cat OPAM/downloads-repo/lib/foo-git-files | sort
+opam
+root.ml
+### cat OPAM/downloads-repo/lib/foo-git-hist
+2
+### :: pinning
+### opam switch create downloads-pin --empty
+### opam pin foo-arch-pin arch.tgz -y
+Package foo-arch-pin does not exist, create as a NEW package? [y/n] y
+[foo-arch-pin.dev] synchronised (file://${BASEDIR}/arch.tgz)
+foo-arch-pin is now pinned to file://${BASEDIR}/arch.tgz (version dev)
+
+The following actions will be performed:
+=== install 1 package
+  - install foo-arch-pin dev (pinned)
+
+<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+-> retrieved foo-arch-pin.dev  (file://${BASEDIR}/arch.tgz)
+-> installed foo-arch-pin.dev
+Done.
+### cat OPAM/downloads-pin/lib/foo-arch-pin-files | sort
+files
+opam
+root.ml
+### opam pin foo-git-pin ./a-dev -y
+Package foo-git-pin does not exist, create as a NEW package? [y/n] y
+[foo-git-pin.dev] synchronised (file://${BASEDIR}/a-dev)
+foo-git-pin is now pinned to git+file://${BASEDIR}/a-dev#master (version dev)
+
+The following actions will be performed:
+=== install 1 package
+  - install foo-git-pin dev (pinned)
+
+<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+-> retrieved foo-git-pin.dev  (git+file://${BASEDIR}/a-dev#master)
+-> installed foo-git-pin.dev
+Done.
+### cat OPAM/downloads-pin/lib/foo-git-pin-files | sort
+opam
+root.ml
+### cat OPAM/downloads-pin/lib/foo-git-pin-hist
+2
+### :: dev-repo
+### opam switch create downloads-devrepo --empty
+### opam pin foo-git --dev-repo -y
+[foo-git.1] synchronised (file://${BASEDIR}/a-dev)
+foo-git is now pinned to git+file://${BASEDIR}/a-dev (version 1)
+
+The following actions will be performed:
+=== install 1 package
+  - install foo-git 1 (pinned)
+
+<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+-> retrieved foo-git.1  (git+file://${BASEDIR}/a-dev)
+-> installed foo-git.1
+Done.
+### cat OPAM/downloads-devrepo/lib/foo-git-files | sort
+opam
+root.ml
+### cat OPAM/downloads-devrepo/lib/foo-git-hist
+2

tests/reftests/source.test

@@ -132,7 +132,7 @@ Now run 'opam upgrade' to apply any package updates.
 ### opam source pandore.4 --dir pandore5
 Successfully extracted to ${BASEDIR}/pandore5
 ### git -C pandore5 rev-list --all --count
-1
+2
 ### opam source pandore.4 --dev --dir pandore6
 Successfully fetched pandore development repo to ${BASEDIR}/pandore6
 ### git -C pandore6 rev-list --all --count