Add separate cache for getPkgConfigDb

Querying pkg-config for the version of every module can be a very expensive operation on some systems. This change adds a separate, per-project, cache for PkgConfigDB; reducing the cost from "every plan change" to "every pkg-config-db change per project". The cache key is composed by the pkg-config configured program and the list of directories reported by pkg-config's pc_path variable. A notice is presented to the user when refreshing the PkgConfigDB.
haskell · Nov 21, 2023 · 03e8346 · 03e8346
1 parent c97092f
commit 03e8346
Show file tree

Hide file tree

Showing 148 changed files with 265 additions and 48 deletions.
diff --git a/cabal-install/src/Distribution/Client/ProjectConfig.hs b/cabal-install/src/Distribution/Client/ProjectConfig.hs
@@ -50,6 +50,7 @@ module Distribution.Client.ProjectConfig
   , resolveSolverSettings
   , BuildTimeSettings (..)
   , resolveBuildTimeSettings
+  , resolveProgramDb
 
     -- * Checking configuration
   , checkBadPerPackageCompilerPaths
@@ -154,6 +155,14 @@ import Distribution.Simple.InstallDirs
   )
 import Distribution.Simple.Program
   ( ConfiguredProgram (..)
+  , ProgramSearchPathEntry (..)
+  )
+import Distribution.Simple.Program.Db
+  ( ProgramDb
+  , defaultProgramDb
+  , modifyProgramSearchPath
+  , userSpecifyArgss
+  , userSpecifyPaths
   )
 import Distribution.Simple.Setup
   ( Flag (Flag)
@@ -496,6 +505,19 @@ resolveBuildTimeSettings
         | isParallelBuild buildSettingNumJobs = False
         | otherwise = False
 
+-- | ProgramDb with directly user specified paths
+resolveProgramDb :: ProjectConfig -> ProgramDb
+resolveProgramDb ProjectConfig{projectConfigLocalPackages = PackageConfig{..}} =
+  userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
+    . userSpecifyArgss (Map.toList (getMapMappend packageConfigProgramArgs))
+    . modifyProgramSearchPath
+      ( ++
+          [ ProgramSearchPathDir dir
+          | dir <- fromNubList packageConfigProgramPathExtra
+          ]
+      )
+    $ defaultProgramDb
+
 ---------------------------------------------
 -- Reading and writing project config files
 --

diff --git a/cabal-install/src/Distribution/Client/ProjectPlanning.hs b/cabal-install/src/Distribution/Client/ProjectPlanning.hs
@@ -481,7 +481,7 @@ configureCompiler
   DistDirLayout
     { distProjectCacheFile
     }
-  ProjectConfig
+  projectConfig@ProjectConfig
     { projectConfigShared =
       ProjectConfigShared
         { projectConfigHcFlavor
@@ -531,15 +531,7 @@ configureCompiler
       hcFlavor = flagToMaybe projectConfigHcFlavor
       hcPath = flagToMaybe projectConfigHcPath
       hcPkg = flagToMaybe projectConfigHcPkg
-      progdb =
-        userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
-          . modifyProgramSearchPath
-            ( [ ProgramSearchPathDir dir
-              | dir <- fromNubList packageConfigProgramPathExtra
-              ]
-                ++
-            )
-          $ defaultProgramDb
+      progdb = resolveProgramDb projectConfig
 
 ------------------------------------------------------------------------------
 
@@ -588,6 +580,9 @@ rebuildInstallPlan
       progsearchpath <- liftIO $ getSystemSearchPath
       let projectConfigMonitored = projectConfig{projectConfigBuildOnly = mempty}
 
+      let progdb = resolveProgramDb projectConfig
+      pkgConfigDB <- getPkgConfigDb verbosity distDirLayout progdb
+
       -- The overall improved plan is cached
       rerunIfChanged
         verbosity
@@ -607,11 +602,12 @@ rebuildInstallPlan
               )
               $ do
                 compilerEtc <- phaseConfigureCompiler projectConfig
-                _ <- phaseConfigurePrograms projectConfig compilerEtc
-                (solverPlan, pkgConfigDB, totalIndexState, activeRepos) <-
+
+                (solverPlan, totalIndexState, activeRepos) <-
                   phaseRunSolver
                     projectConfig
                     compilerEtc
+                    pkgConfigDB
                     localPackages
                     (fromMaybe mempty mbInstalledPackages)
                 ( elaboratedPlan
@@ -650,55 +646,37 @@ rebuildInstallPlan
       phaseConfigureCompiler
         :: ProjectConfig
         -> Rebuild (Compiler, Platform, ProgramDb)
-      phaseConfigureCompiler = configureCompiler verbosity distDirLayout
+      phaseConfigureCompiler projectConfig = do
+        (compiler, platform, progdb) <- configureCompiler verbosity distDirLayout projectConfig
 
-      -- Configuring other programs.
-      --
-      -- Having configred the compiler, now we configure all the remaining
-      -- programs. This is to check we can find them, and to monitor them for
-      -- changes.
-      --
-      -- TODO: [required eventually] we don't actually do this yet.
-      --
-      -- We rely on the fact that the previous phase added the program config for
-      -- all local packages, but that all the programs configured so far are the
-      -- compiler program or related util programs.
-      --
-      phaseConfigurePrograms
-        :: ProjectConfig
-        -> (Compiler, Platform, ProgramDb)
-        -> Rebuild ()
-      phaseConfigurePrograms projectConfig (_, _, compilerprogdb) = do
         -- Users are allowed to specify program locations independently for
         -- each package (e.g. to use a particular version of a pre-processor
         -- for some packages). However they cannot do this for the compiler
         -- itself as that's just not going to work. So we check for this.
         liftIO $
           checkBadPerPackageCompilerPaths
-            (configuredPrograms compilerprogdb)
+            (configuredPrograms progdb)
             (getMapMappend (projectConfigSpecificPackage projectConfig))
 
-      -- TODO: [required eventually] find/configure other programs that the
-      -- user specifies.
-
-      -- TODO: [required eventually] find/configure all build-tools
-      -- but note that some of them may be built as part of the plan.
+        return (compiler, platform, progdb)
 
       -- Run the solver to get the initial install plan.
       -- This is expensive so we cache it independently.
       --
       phaseRunSolver
         :: ProjectConfig
         -> (Compiler, Platform, ProgramDb)
+        -> PkgConfigDb
         -> [PackageSpecifier UnresolvedSourcePackage]
         -> InstalledPackageIndex
-        -> Rebuild (SolverInstallPlan, PkgConfigDb, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
+        -> Rebuild (SolverInstallPlan, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
       phaseRunSolver
         projectConfig@ProjectConfig
           { projectConfigShared
           , projectConfigBuildOnly
           }
         (compiler, platform, progdb)
+        pkgConfigDB
         localPackages
         installedPackages =
           rerunIfChanged
@@ -719,13 +697,13 @@ rebuildInstallPlan
                   progdb
                   platform
                   corePackageDbs
+
               (sourcePkgDb, tis, ar) <-
                 getSourcePackages
                   verbosity
                   withRepoCtx
                   (solverSettingIndexState solverSettings)
                   (solverSettingActiveRepos solverSettings)
-              pkgConfigDB <- getPkgConfigDb verbosity progdb
 
               -- TODO: [code cleanup] it'd be better if the Compiler contained the
               -- ConfiguredPrograms that it needs, rather than relying on the progdb
@@ -750,7 +728,7 @@ rebuildInstallPlan
                   Left msg -> do
                     reportPlanningFailure projectConfig compiler platform localPackages
                     dieWithException verbosity $ PhaseRunSolverErr msg
-                  Right plan -> return (plan, pkgConfigDB, tis, ar)
+                  Right plan -> return (plan, tis, ar)
           where
             corePackageDbs :: [PackageDB]
             corePackageDbs =
@@ -1036,13 +1014,21 @@ getSourcePackages verbosity withRepoCtx idxState activeRepos = do
     $ repos
   return sourcePkgDbWithTIS
 
-getPkgConfigDb :: Verbosity -> ProgramDb -> Rebuild PkgConfigDb
-getPkgConfigDb verbosity progdb = do
+getPkgConfigDb :: Verbosity -> DistDirLayout -> ProgramDb -> Rebuild PkgConfigDb
+getPkgConfigDb verbosity distDirLayout progdb = do
+  -- We find the pkg-config program and we ask it for the location of the pc files.
+  -- We then use these two values for the cache key
+  mpkgConfig <- liftIO $ needProgram verbosity pkgConfigProgram progdb
+  let mpkgConfigLoc = fst <$> mpkgConfig
   dirs <- liftIO $ getPkgConfigDbDirs verbosity progdb
-  -- Just monitor the dirs so we'll notice new .pc files.
-  -- Alternatively we could monitor all the .pc files too.
-  traverse_ monitorDirectoryStatus dirs
-  liftIO $ readPkgConfigDb verbosity progdb
+  rerunIfChanged verbosity fileMonitorPkgConfigDb (mpkgConfigLoc, dirs) $ do
+    liftIO $ notice verbosity "Querying pkg-config database..."
+    -- Just monitor the dirs so we'll notice new .pc files.
+    -- Alternatively we could monitor all the .pc files too.
+    traverse_ monitorDirectoryStatus dirs
+    liftIO $ readPkgConfigDb verbosity progdb
+  where
+    fileMonitorPkgConfigDb = newFileMonitor $ distProjectCacheFile distDirLayout "pkg-config-db"
 
 -- | Select the config values to monitor for changes package source hashes.
 packageLocationsSignature

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external-target.out b/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external-target.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external.out b/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal-target.out b/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal-target.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.out b/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.out b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.out b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:
@@ -18,8 +19,7 @@ Configuring library 'indef' instantiated with
   Data.Map = containers-<VERSION>:Data.Map
 for Includes3-0.1.0.0...
 Preprocessing library 'indef' for Includes3-0.1.0.0...
-Building library 'indef' instantiated with
-  Data.Map = containers-<VERSION>:Data.Map
+Building library 'indef' instantiated with Data.Map = containers-<VERSION>:Data.Map
 for Includes3-0.1.0.0...
 Configuring executable 'exe' for Includes3-0.1.0.0...
 Preprocessing executable 'exe' for Includes3-0.1.0.0...

diff --git a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-repo.out b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-repo.out
@@ -1,6 +1,7 @@
 # cabal v2-update
 Downloading the latest package list from test-local-repo
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/Reexport2/cabal.out b/cabal-testsuite/PackageTests/Backpack/Reexport2/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Error:
     Problem with module re-exports:

diff --git a/cabal-testsuite/PackageTests/Backpack/T6385/cabal.out b/cabal-testsuite/PackageTests/Backpack/T6385/cabal.out
@@ -1,6 +1,7 @@
 # cabal v2-update
 Downloading the latest package list from test-local-repo
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Backpack/bkpcabal01/cabal.out b/cabal-testsuite/PackageTests/Backpack/bkpcabal01/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/BuildDeps/DepCycle/cabal.out b/cabal-testsuite/PackageTests/BuildDeps/DepCycle/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Error:
     Dependency cycle between the following components:

diff --git a/cabal-testsuite/PackageTests/BuildDeps/InternalLibrary1/cabal.out b/cabal-testsuite/PackageTests/BuildDeps/InternalLibrary1/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/BuildTargets/UseLocalPackage/use-local-version-of-package.out b/cabal-testsuite/PackageTests/BuildTargets/UseLocalPackage/use-local-version-of-package.out
@@ -1,6 +1,7 @@
 # cabal v2-update
 Downloading the latest package list from test-local-repo
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/...uite/PackageTests/BuildTargets/UseLocalPackageForSetup/use-local-package-as-setup-dep.out b/...uite/PackageTests/BuildTargets/UseLocalPackageForSetup/use-local-package-as-setup-dep.out
@@ -1,6 +1,7 @@
 # cabal v2-update
 Downloading the latest package list from test-local-repo
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Error: [Cabal-7107]
 Could not resolve dependencies:

diff --git a/cabal-testsuite/PackageTests/BuildToolDepends/setup.out b/cabal-testsuite/PackageTests/BuildToolDepends/setup.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/BuildTools/External/cabal.out b/cabal-testsuite/PackageTests/BuildTools/External/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/BuildTools/Internal/cabal.out b/cabal-testsuite/PackageTests/BuildTools/Internal/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CmmSources/cabal.out b/cabal-testsuite/PackageTests/CmmSources/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-run
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CmmSourcesDyn/cabal.out b/cabal-testsuite/PackageTests/CmmSourcesDyn/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-run
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CmmSourcesExe/cabal.out b/cabal-testsuite/PackageTests/CmmSourcesExe/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-run
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/ConditionalAndImport/cabal.out b/cabal-testsuite/PackageTests/ConditionalAndImport/cabal.out
@@ -1,6 +1,7 @@
 # cabal v2-update
 Downloading the latest package list from test-local-repo
 # cabal v2-run
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/ConfigFile/T8487/cabal.out b/cabal-testsuite/PackageTests/ConfigFile/T8487/cabal.out
@@ -1,4 +1,5 @@
 # cabal build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O2
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Configure/cabal.out b/cabal-testsuite/PackageTests/Configure/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CopyHie/cabal.out b/cabal-testsuite/PackageTests/CopyHie/cabal.out
@@ -1,6 +1,7 @@
 # cabal v2-update
 Downloading the latest package list from test-local-repo
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CustomSegfault/cabal.out b/cabal-testsuite/PackageTests/CustomSegfault/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CustomWithoutCabal/cabal.out b/cabal-testsuite/PackageTests/CustomWithoutCabal/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/CustomWithoutCabalDefaultMain/cabal.out b/cabal-testsuite/PackageTests/CustomWithoutCabalDefaultMain/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built:

diff --git a/cabal-testsuite/PackageTests/Exec/cabal.out b/cabal-testsuite/PackageTests/Exec/cabal.out
@@ -1,4 +1,5 @@
 # cabal v2-build
+Querying pkg-config database...
 Resolving dependencies...
 Build profile: -w ghc-<GHCVER> -O1
 In order, the following will be built: