diff --git a/local_install.go b/local_install.go index e7a51012a..4672aabba 100644 --- a/local_install.go +++ b/local_install.go @@ -51,7 +51,7 @@ func installLocalPKGBUILD( grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, os.Stdout) - graph, err := grapher.GraphFromSrcInfo(nil, wd, pkgbuild) + graph, err := grapher.GraphFromSrcInfo(ctx, nil, wd, pkgbuild) if err != nil { return err } diff --git a/pkg/cmd/graph/main.go b/pkg/cmd/graph/main.go index 19ed8df5d..e0d5630b2 100644 --- a/pkg/cmd/graph/main.go +++ b/pkg/cmd/graph/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "os" "path/filepath" @@ -66,7 +67,7 @@ func handleCmd() error { grapher := dep.NewGrapher(dbExecutor, aurCache, true, settings.NoConfirm, os.Stdout) - return graphPackage(grapher, cmdArgs.Targets) + return graphPackage(context.Background(), grapher, cmdArgs.Targets) } func main() { @@ -77,6 +78,7 @@ func main() { } func graphPackage( + ctx context.Context, grapher *dep.Grapher, targets []string, ) error { @@ -84,7 +86,7 @@ func graphPackage( return errors.New(gotext.Get("only one target is allowed")) } - graph, err := grapher.GraphFromAURCache(nil, []string{targets[0]}) + graph, err := grapher.GraphFromAURCache(ctx, nil, []string{targets[0]}) if err != nil { return err } diff --git a/pkg/dep/depGraph.go b/pkg/dep/depGraph.go index 736dbd03e..5ec6d9483 100644 --- a/pkg/dep/depGraph.go +++ b/pkg/dep/depGraph.go @@ -1,6 +1,7 @@ package dep import ( + "context" "fmt" "io" "os" @@ -106,7 +107,7 @@ func NewGrapher(dbExecutor db.Executor, aurCache *metadata.AURCache, } } -func (g *Grapher) GraphFromTargets(graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) { +func (g *Grapher) GraphFromTargets(ctx context.Context, graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) { if graph == nil { graph = topo.New[string, *InstallInfo]() } @@ -119,7 +120,7 @@ func (g *Grapher) GraphFromTargets(graph *topo.Graph[string, *InstallInfo], targ switch target.DB { case "aur": - graph, err = g.GraphFromAURCache(graph, []string{target.Name}) + graph, err = g.GraphFromAURCache(ctx, graph, []string{target.Name}) default: graph.AddNode(target.Name) g.ValidateAndSetNodeInfo(graph, target.Name, &topo.NodeInfo[*InstallInfo]{ @@ -143,7 +144,7 @@ func (g *Grapher) GraphFromTargets(graph *topo.Graph[string, *InstallInfo], targ return graph, nil } -func (g *Grapher) GraphFromSrcInfo(graph *topo.Graph[string, *InstallInfo], pkgBuildDir string, +func (g *Grapher) GraphFromSrcInfo(ctx context.Context, graph *topo.Graph[string, *InstallInfo], pkgBuildDir string, pkgbuild *gosrc.Srcinfo, ) (*topo.Graph[string, *InstallInfo], error) { if graph == nil { @@ -170,33 +171,33 @@ func (g *Grapher) GraphFromSrcInfo(graph *topo.Graph[string, *InstallInfo], pkgB }, }) - g.addDepNodes(&pkg, graph) + g.addDepNodes(ctx, &pkg, graph) } return graph, nil } -func (g *Grapher) addDepNodes(pkg *aur.Pkg, graph *topo.Graph[string, *InstallInfo]) { +func (g *Grapher) addDepNodes(ctx context.Context, pkg *aur.Pkg, graph *topo.Graph[string, *InstallInfo]) { if len(pkg.MakeDepends) > 0 { - g.addNodes(graph, pkg.Name, pkg.MakeDepends, MakeDep) + g.addNodes(ctx, graph, pkg.Name, pkg.MakeDepends, MakeDep) } if !false && len(pkg.Depends) > 0 { - g.addNodes(graph, pkg.Name, pkg.Depends, Dep) + g.addNodes(ctx, graph, pkg.Name, pkg.Depends, Dep) } if !false && len(pkg.CheckDepends) > 0 { - g.addNodes(graph, pkg.Name, pkg.CheckDepends, CheckDep) + g.addNodes(ctx, graph, pkg.Name, pkg.CheckDepends, CheckDep) } } -func (g *Grapher) GraphFromAURCache(graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) { +func (g *Grapher) GraphFromAURCache(ctx context.Context, graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) { if graph == nil { graph = topo.New[string, *InstallInfo]() } for _, target := range targets { - aurPkgs, _ := g.aurCache.FindPackage(target) + aurPkgs, _ := g.aurCache.FindPackage(ctx, target) if len(aurPkgs) == 0 { text.Errorln("No AUR package found for", target) @@ -217,7 +218,7 @@ func (g *Grapher) GraphFromAURCache(graph *topo.Graph[string, *InstallInfo], tar }) graph.AddNode(pkg.Name) - g.addDepNodes(pkg, graph) + g.addDepNodes(ctx, pkg, graph) } return graph, nil @@ -237,6 +238,7 @@ func (g *Grapher) ValidateAndSetNodeInfo(graph *topo.Graph[string, *InstallInfo] } func (g *Grapher) addNodes( + ctx context.Context, graph *topo.Graph[string, *InstallInfo], parentPkgName string, deps []string, @@ -295,13 +297,13 @@ func (g *Grapher) addNodes( newDepsSlice = append(newDepsSlice, newDep.Name) } - g.addNodes(graph, alpmPkg.Name(), newDepsSlice, Dep) + g.addNodes(ctx, graph, alpmPkg.Name(), newDepsSlice, Dep) } continue } - if aurPkgs, _ := g.aurCache.FindPackage(depName); len(aurPkgs) != 0 { // Check AUR + if aurPkgs, _ := g.aurCache.FindPackage(ctx, depName); len(aurPkgs) != 0 { // Check AUR pkg := aurPkgs[0] if len(aurPkgs) > 1 { pkg = provideMenu(g.w, depName, aurPkgs, g.noConfirm) @@ -324,7 +326,7 @@ func (g *Grapher) addNodes( Version: pkg.Version, }, }) - g.addDepNodes(pkg, graph) + g.addDepNodes(ctx, pkg, graph) continue } diff --git a/pkg/metadata/metadata_aur.go b/pkg/metadata/metadata_aur.go index 6d5401290..a946e78a1 100644 --- a/pkg/metadata/metadata_aur.go +++ b/pkg/metadata/metadata_aur.go @@ -1,22 +1,38 @@ package metadata import ( + "context" "encoding/json" "fmt" "log" + "os" + "time" "github.com/Jguer/aur" "github.com/itchyny/gojq" "github.com/ohler55/ojg/oj" - "github.com/tidwall/gjson" +) + +const ( + searchCacheCap = 300 + cacheValidity = 1 * time.Hour ) type AURCache struct { cache []byte - provideCache map[string][]*aur.Pkg + searchCache map[string][]*aur.Pkg + cachePath string unmarshalledCache []interface{} cacheHits int gojqCode *gojq.Code + DebugLoggerFn func(a ...interface{}) +} + +type AURQuery struct { + ByProvides bool // Returns multiple results of different bases + ByBase bool // Returns multiple results of the same base + ByName bool // Returns only 1 or 0 results + Needles []string } func NewAURCache(cachePath string) (*AURCache, error) { @@ -28,74 +44,161 @@ func NewAURCache(cachePath string) (*AURCache, error) { return &AURCache{ cache: aurCache, - provideCache: make(map[string][]*aur.Pkg, 300), + cachePath: cachePath, + searchCache: make(map[string][]*aur.Pkg, searchCacheCap), unmarshalledCache: inputStruct.([]interface{}), gojqCode: makeGoJQ(), }, nil } +// needsUpdate checks if cachepath is older than 24 hours +func (a *AURCache) needsUpdate() (bool, error) { + // check if cache is older than 24 hours + info, err := os.Stat(a.cachePath) + if err != nil { + return false, fmt.Errorf("unable to read cache: %w", err) + } + + return info.ModTime().Before(time.Now().Add(-cacheValidity)), nil +} + +func (a *AURCache) cacheKey(needle string, byProvides, byBase, byName bool) string { + return fmt.Sprintf("%s-%v-%v-%v", needle, byProvides, byBase, byName) +} + func (a *AURCache) DebugInfo() { fmt.Println("Byte Cache", len(a.cache)) - fmt.Println("Entries Cached", len(a.provideCache)) + fmt.Println("Entries Cached", len(a.searchCache)) fmt.Println("Cache Hits", a.cacheHits) } func (a *AURCache) SetProvideCache(needle string, pkgs []*aur.Pkg) { - a.provideCache[needle] = pkgs + a.searchCache[needle] = pkgs +} + +// Get returns a list of packages that provide the given search term. +func (a *AURCache) Get(ctx context.Context, query *AURQuery) ([]*aur.Pkg, error) { + update, err := a.needsUpdate() + if err != nil { + return nil, err + } + + if update { + if a.DebugLoggerFn != nil { + a.DebugLoggerFn("AUR Cache is out of date, updating") + } + + var makeErr error + if a.cache, makeErr = MakeCache(a.cachePath); makeErr != nil { + return nil, makeErr + } + + inputStruct, unmarshallErr := oj.Parse(a.cache) + if unmarshallErr != nil { + return nil, unmarshallErr + } + + a.unmarshalledCache = inputStruct.([]interface{}) + } + + found := make([]*aur.Pkg, 0, len(query.Needles)) + if len(query.Needles) == 0 { + return found, nil + } + + iterFound, errNeedle := a.gojqGetBatch(ctx, query) + if errNeedle != nil { + return nil, errNeedle + } + + found = append(found, iterFound...) + + return found, nil } // Get returns a list of packages that provide the given search term -func (a *AURCache) FindPackage(needle string) ([]*aur.Pkg, error) { - if pkgs, ok := a.provideCache[needle]; ok { +func (a *AURCache) FindPackage(ctx context.Context, needle string) ([]*aur.Pkg, error) { + cacheKey := a.cacheKey(needle, true, true, true) + if pkgs, ok := a.searchCache[cacheKey]; ok { a.cacheHits++ return pkgs, nil } - final, error := a.gojqGet(needle) + final, error := a.gojqGet(ctx, needle) if error != nil { return nil, error } - a.provideCache[needle] = final + a.searchCache[cacheKey] = final return final, nil } -func (a *AURCache) gjsonGet(depName string) ([]*aur.Pkg, error) { - dedupMap := make(map[string]bool) - queryProvides := fmt.Sprintf("#(Provides.#(==\"%s\"))#", depName) - queryNames := fmt.Sprintf("#(Name==\"%s\")#", depName) - queryBases := fmt.Sprintf("#(PackageBase==\"%s\")#", depName) +func (a *AURCache) gojqGetBatch(ctx context.Context, query *AURQuery) ([]*aur.Pkg, error) { + pattern := ".[] | select(" + for i, searchTerm := range query.Needles { + if i != 0 { + pattern += " or " + } + + if query.ByName { + pattern += fmt.Sprintf("(.Name == \"%s\")", searchTerm) + if query.ByBase || query.ByProvides { + pattern += " or " + } + } + + if query.ByBase { + pattern += fmt.Sprintf("(.PackageBase == \"%s\")", searchTerm) + if query.ByProvides { + pattern += " or " + } + } + + if query.ByProvides { + pattern += fmt.Sprintf("(.Provides[]? == \"%s\")", searchTerm) + } + } - results := gjson.GetManyBytes(a.cache, queryProvides, queryNames, queryBases) + pattern += ")" - aggregated := append(append(results[0].Array(), results[1].Array()...), results[2].Array()...) + parsed, err := gojq.Parse(pattern) + if err != nil { + log.Fatalln(err) + } - final := make([]*aur.Pkg, 0, len(aggregated)) + final := make([]*aur.Pkg, 0, len(query.Needles)) - for i := range aggregated { - jsonString := aggregated[i].Raw - key := jsonString[:15] + iter := parsed.RunWithContext(ctx, a.unmarshalledCache) // or query.RunWithContext - if _, ok := dedupMap[key]; !ok { - pkg := &aur.Pkg{} - json.Unmarshal([]byte(jsonString), pkg) - final = append(final, pkg) - dedupMap[key] = true + for v, ok := iter.Next(); ok; v, ok = iter.Next() { + if err, ok := v.(error); ok { + return nil, err } + + pkg := new(aur.Pkg) + bValue, err := gojq.Marshal(v) + if err != nil { + log.Fatalln(err) + } + + oj.Unmarshal(bValue, pkg) + final = append(final, pkg) + } + + if a.DebugLoggerFn != nil { + a.DebugLoggerFn("AUR Query", pattern, "Found", len(final)) } + return final, nil } -func (a *AURCache) gojqGet(searchTerm string) ([]*aur.Pkg, error) { +func (a *AURCache) gojqGet(ctx context.Context, searchTerm string) ([]*aur.Pkg, error) { final := make([]*aur.Pkg, 0, 1) - iter := a.gojqCode.Run(a.unmarshalledCache, searchTerm) // or query.RunWithContext - for { - v, ok := iter.Next() - if !ok { - break - } + iter := a.gojqCode.RunWithContext(ctx, a.unmarshalledCache, searchTerm) // or query.RunWithContext + + for v, ok := iter.Next(); ok; v, ok = iter.Next() { if err, ok := v.(error); ok { return nil, err } diff --git a/pkg/metadata/metadata_downloader.go b/pkg/metadata/metadata_downloader.go index 02f94a25a..388c516b4 100644 --- a/pkg/metadata/metadata_downloader.go +++ b/pkg/metadata/metadata_downloader.go @@ -7,24 +7,6 @@ import ( "os" ) -func main() { - // check if cache exists - cachePath := "aur.json" - cacheBytes, err := ReadCache(cachePath) - if err != nil { - fmt.Println(err) - return - } - - if len(cacheBytes) == 0 { - cacheBytes, err = MakeCache(cachePath) - if err != nil { - fmt.Println(err) - return - } - } -} - func MakeOrReadCache(cachePath string) ([]byte, error) { cacheBytes, err := ReadCache(cachePath) if err != nil { diff --git a/pkg/settings/config.go b/pkg/settings/config.go index 7330abd72..1ae19feaf 100644 --- a/pkg/settings/config.go +++ b/pkg/settings/config.go @@ -296,6 +296,8 @@ func NewConfig(version string) (*Configuration, error) { return nil, errors.Wrap(errAURCache, gotext.Get("failed to retrieve aur Cache")) } + newConfig.Runtime.AURCache.DebugLoggerFn = text.Debugln + var errAUR error newConfig.Runtime.AURClient, errAUR = aur.NewClient(aur.WithHTTPClient(newConfig.Runtime.HTTPClient), aur.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error { diff --git a/print.go b/print.go index 5bfcd9edd..52b5782d4 100644 --- a/print.go +++ b/print.go @@ -104,7 +104,7 @@ func printNumberOfUpdates(ctx context.Context, dbExecutor db.Executor, enableDow warnings := query.NewWarnings() old := os.Stdout // keep backup of the real stdout os.Stdout = nil - aurUp, repoUp, err := upList(ctx, warnings, dbExecutor, enableDowngrade, filter) + aurUp, repoUp, err := upList(ctx, nil, warnings, dbExecutor, enableDowngrade, filter) os.Stdout = old // restoring the real stdout if err != nil { @@ -117,7 +117,8 @@ func printNumberOfUpdates(ctx context.Context, dbExecutor db.Executor, enableDow } func printUpdateList(ctx context.Context, cmdArgs *parser.Arguments, - dbExecutor db.Executor, enableDowngrade bool, filter upgrade.Filter) error { + dbExecutor db.Executor, enableDowngrade bool, filter upgrade.Filter, +) error { targets := stringset.FromSlice(cmdArgs.Targets) warnings := query.NewWarnings() old := os.Stdout // keep backup of the real stdout @@ -129,7 +130,7 @@ func printUpdateList(ctx context.Context, cmdArgs *parser.Arguments, return err } - aurUp, repoUp, err := upList(ctx, warnings, dbExecutor, enableDowngrade, filter) + aurUp, repoUp, err := upList(ctx, nil, warnings, dbExecutor, enableDowngrade, filter) os.Stdout = old // restoring the real stdout if err != nil { diff --git a/sync.go b/sync.go index 1a6a9ce09..dc12a5c99 100644 --- a/sync.go +++ b/sync.go @@ -35,14 +35,14 @@ func syncInstall(ctx context.Context, grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, os.Stdout) - graph, err := grapher.GraphFromTargets(nil, cmdArgs.Targets) + graph, err := grapher.GraphFromTargets(ctx, nil, cmdArgs.Targets) if err != nil { return err } if cmdArgs.ExistsArg("u", "sysupgrade") { var errSysUp error - graph, _, errSysUp = sysupgradeTargetsV2(ctx, dbExecutor, graph, cmdArgs.ExistsDouble("u", "sysupgrade")) + graph, _, errSysUp = sysupgradeTargetsV2(ctx, aurCache, dbExecutor, graph, cmdArgs.ExistsDouble("u", "sysupgrade")) if errSysUp != nil { return errSysUp } diff --git a/upgrade.go b/upgrade.go index 17563aa10..db82e5a43 100644 --- a/upgrade.go +++ b/upgrade.go @@ -14,6 +14,7 @@ import ( "github.com/Jguer/yay/v11/pkg/db" "github.com/Jguer/yay/v11/pkg/dep" "github.com/Jguer/yay/v11/pkg/intrange" + "github.com/Jguer/yay/v11/pkg/metadata" "github.com/Jguer/yay/v11/pkg/multierror" "github.com/Jguer/yay/v11/pkg/query" "github.com/Jguer/yay/v11/pkg/settings" @@ -36,7 +37,8 @@ func filterUpdateList(list []db.Upgrade, filter upgrade.Filter) []db.Upgrade { } // upList returns lists of packages to upgrade from each source. -func upList(ctx context.Context, warnings *query.AURWarnings, dbExecutor db.Executor, enableDowngrade bool, +func upList(ctx context.Context, aurCache *metadata.AURCache, + warnings *query.AURWarnings, dbExecutor db.Executor, enableDowngrade bool, filter upgrade.Filter, ) (aurUp, repoUp upgrade.UpSlice, err error) { remote, remoteNames := query.GetRemotePackages(dbExecutor) @@ -71,7 +73,11 @@ func upList(ctx context.Context, warnings *query.AURWarnings, dbExecutor db.Exec text.OperationInfoln(gotext.Get("Searching AUR for updates...")) var _aurdata []*aur.Pkg - _aurdata, err = query.AURInfo(ctx, config.Runtime.AURClient, remoteNames, warnings, config.RequestSplitN) + if aurCache != nil { + _aurdata, err = aurCache.Get(ctx, &metadata.AURQuery{ByName: true, Needles: remoteNames}) + } else { + _aurdata, err = query.AURInfo(ctx, config.Runtime.AURClient, remoteNames, warnings, config.RequestSplitN) + } errs.Add(err) if err == nil { @@ -241,7 +247,7 @@ func sysupgradeTargets(ctx context.Context, dbExecutor db.Executor, ) (stringset.StringSet, []string, error) { warnings := query.NewWarnings() - aurUp, repoUp, err := upList(ctx, warnings, dbExecutor, enableDowngrade, + aurUp, repoUp, err := upList(ctx, nil, warnings, dbExecutor, enableDowngrade, func(upgrade.Upgrade) bool { return true }) if err != nil { return nil, nil, err @@ -254,13 +260,14 @@ func sysupgradeTargets(ctx context.Context, dbExecutor db.Executor, // Targets for sys upgrade. func sysupgradeTargetsV2(ctx context.Context, + aurCache *metadata.AURCache, dbExecutor db.Executor, graph *topo.Graph[string, *dep.InstallInfo], enableDowngrade bool, ) (*topo.Graph[string, *dep.InstallInfo], stringset.StringSet, error) { warnings := query.NewWarnings() - aurUp, repoUp, err := upList(ctx, warnings, dbExecutor, enableDowngrade, + aurUp, repoUp, err := upList(ctx, aurCache, warnings, dbExecutor, enableDowngrade, func(upgrade.Upgrade) bool { return true }) if err != nil { return graph, nil, err