From fa7609950023462c6f91c425de7610c0bb6b86ba Mon Sep 17 00:00:00 2001 From: wrvsrx <42770726+wrvsrx@users.noreply.github.com> Date: Thu, 13 Jun 2024 12:20:48 +0800 Subject: [PATCH] Support sparseCheckout (#123) * support sparseCheckout * add some workaround for fetchFromGitHub * add a sparseCheckout example * fix git test --- app/Config/PackageFetcher.hs | 10 +++++++--- nvfetcher_example.toml | 6 ++++++ src/NvFetcher/NixExpr.hs | 14 +++++++++----- src/NvFetcher/NixFetcher.hs | 15 ++++++++------- src/NvFetcher/Types.hs | 10 ++++++++-- test/NixExprSpec.hs | 1 + 6 files changed, 39 insertions(+), 17 deletions(-) diff --git a/app/Config/PackageFetcher.hs b/app/Config/PackageFetcher.hs index 2ff4f9c..f03597e 100644 --- a/app/Config/PackageFetcher.hs +++ b/app/Config/PackageFetcher.hs @@ -50,7 +50,8 @@ fetcherKeys = data GitOptions = GitOptions { goDeepClone :: Maybe Bool, goFetchSubmodules :: Maybe Bool, - goLeaveDotGit :: Maybe Bool + goLeaveDotGit :: Maybe Bool, + goSparseCheckout :: Maybe [Text] } deriving (Eq, Generic) @@ -60,6 +61,7 @@ gitOptionsDecoder = <$> getFieldsOpt ["git", "deepClone"] <*> getFieldsOpt ["git", "fetchSubmodules"] <*> getFieldsOpt ["git", "leaveDotGit"] + <*> getFieldsOpt ["git", "sparseCheckout"] _GitOptions :: Traversal' (NixFetcher f) GitOptions _GitOptions f x@FetchGit {..} = @@ -68,16 +70,18 @@ _GitOptions f x@FetchGit {..} = & deepClone .~ fromMaybe False goDeepClone & fetchSubmodules .~ fromMaybe False goFetchSubmodules & leaveDotGit .~ fromMaybe False goLeaveDotGit + & sparseCheckout .~ fromMaybe [] goSparseCheckout ) - <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit)) + <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit) (Just _sparseCheckout)) _GitOptions f x@FetchGitHub {..} = ( \GitOptions {..} -> x & deepClone .~ fromMaybe False goDeepClone & fetchSubmodules .~ fromMaybe False goFetchSubmodules & leaveDotGit .~ fromMaybe False goLeaveDotGit + & sparseCheckout .~ fromMaybe [] goSparseCheckout ) - <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit)) + <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit) (Just _sparseCheckout)) _GitOptions _ x = pure x -------------------------------------------------------------------------------- diff --git a/nvfetcher_example.toml b/nvfetcher_example.toml index 0d02bfe..691245f 100644 --- a/nvfetcher_example.toml +++ b/nvfetcher_example.toml @@ -68,3 +68,9 @@ fetch.url = "https://files.yande.re/image/3fc51f6a2fb10c96b73dd0fce6ddb9c8/yande src.manual = "latest" # Override the name of the file in the Nix store url.name = "wallpaper.jpg" + +# To demonstrate how to use `sparseCheckout` option +[noto-fonts-cjk-sans-fix-weight] +src.manual = "Sans2.004" +fetch.github = "notofonts/noto-cjk" +git.sparseCheckout = [ "Sans/OTC" ] diff --git a/src/NvFetcher/NixExpr.hs b/src/NvFetcher/NixExpr.hs index cbc4981..022256b 100644 --- a/src/NvFetcher/NixExpr.hs +++ b/src/NvFetcher/NixExpr.hs @@ -67,6 +67,7 @@ nixFetcher = \case _fetchSubmodules = toNixExpr -> fetchSubmodules, _deepClone = toNixExpr -> deepClone, _leaveDotGit = toNixExpr -> leaveDotGit, + _sparseCheckout = toNixExpr . map quote -> sparseCheckout, _furl = quote -> url, _name = nameField -> n } -> @@ -76,7 +77,8 @@ nixFetcher = \case rev = $rev; fetchSubmodules = $fetchSubmodules; deepClone = $deepClone; - leaveDotGit = $leaveDotGit;$n + leaveDotGit = $leaveDotGit; + sparseCheckout = $sparseCheckout;$n sha256 = $sha256; } |] @@ -86,13 +88,14 @@ nixFetcher = \case _fetchSubmodules = toNixExpr -> fetchSubmodules, _deepClone = toNixExpr -> deepClone, _leaveDotGit = toNixExpr -> leaveDotGit, + _sparseCheckout = toNixExpr . map quote -> sparseCheckout, _fowner = quote -> owner, _frepo = quote -> repo, _name = nameField -> n } -> - -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone and - -- leaveDotGit won't get passed to fetchzip - if (deepClone == "true") || (leaveDotGit == "true") + -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone, leaveDotGit + -- and sparseCheckout won't get passed to fetchzip + if (deepClone == "true") || (leaveDotGit == "true") || (sparseCheckout /= "[ ]") then [trimming| fetchFromGitHub { @@ -101,7 +104,8 @@ nixFetcher = \case rev = $rev; fetchSubmodules = $fetchSubmodules; deepClone = $deepClone; - leaveDotGit = $leaveDotGit;$n + leaveDotGit = $leaveDotGit; + sparseCheckout = $sparseCheckout;$n sha256 = $sha256; } |] diff --git a/src/NvFetcher/NixFetcher.hs b/src/NvFetcher/NixFetcher.hs index c69abe8..746b1fd 100644 --- a/src/NvFetcher/NixFetcher.hs +++ b/src/NvFetcher/NixFetcher.hs @@ -97,8 +97,8 @@ runNixPrefetchUrl url unpack name = do newtype FetchedGit = FetchedGit {sha256 :: Text} deriving (Show, Generic, A.FromJSON) -runNixPrefetchGit :: Text -> Text -> Bool -> Bool -> Bool -> Action Checksum -runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do +runNixPrefetchGit :: Text -> Text -> Bool -> Bool -> Bool -> [Text] -> Action Checksum +runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit sparseCheckout = do (CmdTime t, Stdout out, CmdLine c) <- quietly $ command [EchoStderr False] "nix-prefetch-git" $ @@ -107,6 +107,7 @@ runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do <> ["--fetch-submodules" | fetchSubmodules] <> ["--deepClone" | deepClone] <> ["--leave-dotGit" | leaveDotGit] + <> if null sparseCheckout then [] else ["--sparse-checkout", T.unpack $ T.intercalate "\n" sparseCheckout] putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s" case A.eitherDecode out of Right (FetchedGit x) -> sha256ToSri x @@ -117,14 +118,14 @@ runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do runFetcher :: NixFetcher Fresh -> Action (NixFetcher Fetched) runFetcher = \case FetchGit {..} -> do - result <- runNixPrefetchGit _furl (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit + result <- runNixPrefetchGit _furl (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit _sparseCheckout pure FetchGit {_sha256 = coerce result, ..} FetchGitHub {..} -> do - let useFetchGit = _fetchSubmodules || _leaveDotGit || _deepClone + let useFetchGit = _fetchSubmodules || _leaveDotGit || _deepClone || not (null _sparseCheckout) ver = coerce _rev result <- if useFetchGit - then runNixPrefetchGit [trimming|https://github.com/$_fowner/$_frepo|] (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit + then runNixPrefetchGit [trimming|https://github.com/$_fowner/$_frepo|] (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit _sparseCheckout else runNixPrefetchUrl [trimming|https://github.com/$_fowner/$_frepo/archive/$ver.tar.gz|] True mempty pure FetchGitHub {_sha256 = result, ..} FetchUrl {..} -> do @@ -186,14 +187,14 @@ prefetch f force = askOracle $ RunFetch force f -- | Create a fetcher from git url gitFetcher :: Text -> PackageFetcher -gitFetcher furl rev = FetchGit furl rev False True False Nothing () +gitFetcher furl rev = FetchGit furl rev False True False [] Nothing () -- | Create a fetcher from github repo gitHubFetcher :: -- | owner and repo (Text, Text) -> PackageFetcher -gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False Nothing () +gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False [] Nothing () -- | Create a fetcher from pypi pypiFetcher :: Text -> PackageFetcher diff --git a/src/NvFetcher/Types.hs b/src/NvFetcher/Types.hs index f658c1b..e0fc766 100644 --- a/src/NvFetcher/Types.hs +++ b/src/NvFetcher/Types.hs @@ -394,6 +394,7 @@ data NixFetcher (k :: FetchStatus) _deepClone :: Bool, _fetchSubmodules :: Bool, _leaveDotGit :: Bool, + _sparseCheckout :: [Text], _name :: Maybe Text, _sha256 :: FetchResult Checksum k } @@ -404,6 +405,7 @@ data NixFetcher (k :: FetchStatus) _deepClone :: Bool, _fetchSubmodules :: Bool, _leaveDotGit :: Bool, + _sparseCheckout :: [Text], _name :: Maybe Text, _sha256 :: FetchResult Checksum k } @@ -461,6 +463,7 @@ instance A.ToJSON (NixFetcher Fetched) where "deepClone" A..= _deepClone, "fetchSubmodules" A..= _fetchSubmodules, "leaveDotGit" A..= _leaveDotGit, + "sparseCheckout" A..= _sparseCheckout, "name" A..= _name, "sha256" A..= _sha256, "type" A..= A.String "git" @@ -473,6 +476,7 @@ instance A.ToJSON (NixFetcher Fetched) where "deepClone" A..= _deepClone, "fetchSubmodules" A..= _fetchSubmodules, "leaveDotGit" A..= _leaveDotGit, + "sparseCheckout" A..= _sparseCheckout, "name" A..= _name, "sha256" A..= _sha256, "type" A..= A.String "github" @@ -514,7 +518,8 @@ instance Pretty (NixFetcher k) where "rev" <> colon <+> pretty _rev, "deepClone" <> colon <+> pretty _deepClone, "fetchSubmodules" <> colon <+> pretty _fetchSubmodules, - "leaveDotGit" <> colon <+> pretty _leaveDotGit + "leaveDotGit" <> colon <+> pretty _leaveDotGit, + "sparseCheckout" <> colon <+> pretty _sparseCheckout ] <> ppField "name" _name ) @@ -529,7 +534,8 @@ instance Pretty (NixFetcher k) where "rev" <> colon <+> pretty _rev, "deepClone" <> colon <+> pretty _deepClone, "fetchSubmodules" <> colon <+> pretty _fetchSubmodules, - "leaveDotGit" <> colon <+> pretty _leaveDotGit + "leaveDotGit" <> colon <+> pretty _leaveDotGit, + "sparseCheckout" <> colon <+> pretty _sparseCheckout ] <> ppField "name" _name ) diff --git a/test/NixExprSpec.hs b/test/NixExprSpec.hs index 1e8071b..e77660a 100644 --- a/test/NixExprSpec.hs +++ b/test/NixExprSpec.hs @@ -34,6 +34,7 @@ spec = describe "toNixExpr" $ do fetchSubmodules = true; deepClone = false; leaveDotGit = false; + sparseCheckout = [ ]; sha256 = "0000000000000000000000000000000000000000000000000000000000000000"; } |]