Skip to content

Commit

Permalink
use aur cache for upgrades
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jguer
Jguer committed Nov 8, 2022
1 parent 9b576fb commit 742b6ad
Show file tree
Hide file tree
Showing 9 changed files with 175 additions and 76 deletions.
2 changes: 1 addition & 1 deletion local_install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func installLocalPKGBUILD(

grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, os.Stdout)

graph, err := grapher.GraphFromSrcInfo(nil, wd, pkgbuild)
graph, err := grapher.GraphFromSrcInfo(ctx, nil, wd, pkgbuild)
if err != nil {
return err
}
Expand Down
6 changes: 4 additions & 2 deletions pkg/cmd/graph/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"context"
"fmt"
"os"
"path/filepath"
Expand Down Expand Up @@ -66,7 +67,7 @@ func handleCmd() error {

grapher := dep.NewGrapher(dbExecutor, aurCache, true, settings.NoConfirm, os.Stdout)

return graphPackage(grapher, cmdArgs.Targets)
return graphPackage(context.Background(), grapher, cmdArgs.Targets)
}

func main() {
Expand All @@ -77,14 +78,15 @@ func main() {
}

func graphPackage(
ctx context.Context,
grapher *dep.Grapher,
targets []string,
) error {
if len(targets) != 1 {
return errors.New(gotext.Get("only one target is allowed"))
}

graph, err := grapher.GraphFromAURCache(nil, []string{targets[0]})
graph, err := grapher.GraphFromAURCache(ctx, nil, []string{targets[0]})
if err != nil {
return err
}
Expand Down
30 changes: 16 additions & 14 deletions pkg/dep/depGraph.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dep

import (
"context"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -106,7 +107,7 @@ func NewGrapher(dbExecutor db.Executor, aurCache *metadata.AURCache,
}
}

func (g *Grapher) GraphFromTargets(graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) {
func (g *Grapher) GraphFromTargets(ctx context.Context, graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) {
if graph == nil {
graph = topo.New[string, *InstallInfo]()
}
Expand All @@ -119,7 +120,7 @@ func (g *Grapher) GraphFromTargets(graph *topo.Graph[string, *InstallInfo], targ

switch target.DB {
case "aur":
graph, err = g.GraphFromAURCache(graph, []string{target.Name})
graph, err = g.GraphFromAURCache(ctx, graph, []string{target.Name})
default:
graph.AddNode(target.Name)
g.ValidateAndSetNodeInfo(graph, target.Name, &topo.NodeInfo[*InstallInfo]{
Expand All @@ -143,7 +144,7 @@ func (g *Grapher) GraphFromTargets(graph *topo.Graph[string, *InstallInfo], targ
return graph, nil
}

func (g *Grapher) GraphFromSrcInfo(graph *topo.Graph[string, *InstallInfo], pkgBuildDir string,
func (g *Grapher) GraphFromSrcInfo(ctx context.Context, graph *topo.Graph[string, *InstallInfo], pkgBuildDir string,
pkgbuild *gosrc.Srcinfo,
) (*topo.Graph[string, *InstallInfo], error) {
if graph == nil {
Expand All @@ -170,33 +171,33 @@ func (g *Grapher) GraphFromSrcInfo(graph *topo.Graph[string, *InstallInfo], pkgB
},
})

g.addDepNodes(&pkg, graph)
g.addDepNodes(ctx, &pkg, graph)
}

return graph, nil
}

func (g *Grapher) addDepNodes(pkg *aur.Pkg, graph *topo.Graph[string, *InstallInfo]) {
func (g *Grapher) addDepNodes(ctx context.Context, pkg *aur.Pkg, graph *topo.Graph[string, *InstallInfo]) {
if len(pkg.MakeDepends) > 0 {
g.addNodes(graph, pkg.Name, pkg.MakeDepends, MakeDep)
g.addNodes(ctx, graph, pkg.Name, pkg.MakeDepends, MakeDep)
}

if !false && len(pkg.Depends) > 0 {
g.addNodes(graph, pkg.Name, pkg.Depends, Dep)
g.addNodes(ctx, graph, pkg.Name, pkg.Depends, Dep)
}

if !false && len(pkg.CheckDepends) > 0 {
g.addNodes(graph, pkg.Name, pkg.CheckDepends, CheckDep)
g.addNodes(ctx, graph, pkg.Name, pkg.CheckDepends, CheckDep)
}
}

func (g *Grapher) GraphFromAURCache(graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) {
func (g *Grapher) GraphFromAURCache(ctx context.Context, graph *topo.Graph[string, *InstallInfo], targets []string) (*topo.Graph[string, *InstallInfo], error) {
if graph == nil {
graph = topo.New[string, *InstallInfo]()
}

for _, target := range targets {
aurPkgs, _ := g.aurCache.FindPackage(target)
aurPkgs, _ := g.aurCache.FindPackage(ctx, target)
if len(aurPkgs) == 0 {
text.Errorln("No AUR package found for", target)

Expand All @@ -217,7 +218,7 @@ func (g *Grapher) GraphFromAURCache(graph *topo.Graph[string, *InstallInfo], tar
})

graph.AddNode(pkg.Name)
g.addDepNodes(pkg, graph)
g.addDepNodes(ctx, pkg, graph)
}

return graph, nil
Expand All @@ -237,6 +238,7 @@ func (g *Grapher) ValidateAndSetNodeInfo(graph *topo.Graph[string, *InstallInfo]
}

func (g *Grapher) addNodes(
ctx context.Context,
graph *topo.Graph[string, *InstallInfo],
parentPkgName string,
deps []string,
Expand Down Expand Up @@ -295,13 +297,13 @@ func (g *Grapher) addNodes(
newDepsSlice = append(newDepsSlice, newDep.Name)
}

g.addNodes(graph, alpmPkg.Name(), newDepsSlice, Dep)
g.addNodes(ctx, graph, alpmPkg.Name(), newDepsSlice, Dep)
}

continue
}

if aurPkgs, _ := g.aurCache.FindPackage(depName); len(aurPkgs) != 0 { // Check AUR
if aurPkgs, _ := g.aurCache.FindPackage(ctx, depName); len(aurPkgs) != 0 { // Check AUR
pkg := aurPkgs[0]
if len(aurPkgs) > 1 {
pkg = provideMenu(g.w, depName, aurPkgs, g.noConfirm)
Expand All @@ -324,7 +326,7 @@ func (g *Grapher) addNodes(
Version: pkg.Version,
},
})
g.addDepNodes(pkg, graph)
g.addDepNodes(ctx, pkg, graph)

continue
}
Expand Down
167 changes: 135 additions & 32 deletions pkg/metadata/metadata_aur.go
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,38 @@
package metadata

import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"time"

"github.com/Jguer/aur"
"github.com/itchyny/gojq"
"github.com/ohler55/ojg/oj"
"github.com/tidwall/gjson"
)

const (
searchCacheCap = 300
cacheValidity = 1 * time.Hour
)

type AURCache struct {
cache []byte
provideCache map[string][]*aur.Pkg
searchCache map[string][]*aur.Pkg
cachePath string
unmarshalledCache []interface{}
cacheHits int
gojqCode *gojq.Code
DebugLoggerFn func(a ...interface{})
}

type AURQuery struct {
ByProvides bool // Returns multiple results of different bases
ByBase bool // Returns multiple results of the same base
ByName bool // Returns only 1 or 0 results
Needles []string
}

func NewAURCache(cachePath string) (*AURCache, error) {
Expand All @@ -28,74 +44,161 @@ func NewAURCache(cachePath string) (*AURCache, error) {

return &AURCache{
cache: aurCache,
provideCache: make(map[string][]*aur.Pkg, 300),
cachePath: cachePath,
searchCache: make(map[string][]*aur.Pkg, searchCacheCap),
unmarshalledCache: inputStruct.([]interface{}),
gojqCode: makeGoJQ(),
}, nil
}

// needsUpdate checks if cachepath is older than 24 hours
func (a *AURCache) needsUpdate() (bool, error) {
// check if cache is older than 24 hours
info, err := os.Stat(a.cachePath)
if err != nil {
return false, fmt.Errorf("unable to read cache: %w", err)
}

return info.ModTime().Before(time.Now().Add(-cacheValidity)), nil
}

func (a *AURCache) cacheKey(needle string, byProvides, byBase, byName bool) string {
return fmt.Sprintf("%s-%v-%v-%v", needle, byProvides, byBase, byName)
}

func (a *AURCache) DebugInfo() {
fmt.Println("Byte Cache", len(a.cache))
fmt.Println("Entries Cached", len(a.provideCache))
fmt.Println("Entries Cached", len(a.searchCache))
fmt.Println("Cache Hits", a.cacheHits)
}

func (a *AURCache) SetProvideCache(needle string, pkgs []*aur.Pkg) {
a.provideCache[needle] = pkgs
a.searchCache[needle] = pkgs
}

// Get returns a list of packages that provide the given search term.
func (a *AURCache) Get(ctx context.Context, query *AURQuery) ([]*aur.Pkg, error) {
update, err := a.needsUpdate()
if err != nil {
return nil, err
}

if update {
if a.DebugLoggerFn != nil {
a.DebugLoggerFn("AUR Cache is out of date, updating")
}

var makeErr error
if a.cache, makeErr = MakeCache(a.cachePath); makeErr != nil {
return nil, makeErr
}

inputStruct, unmarshallErr := oj.Parse(a.cache)
if unmarshallErr != nil {
return nil, unmarshallErr
}

a.unmarshalledCache = inputStruct.([]interface{})
}

found := make([]*aur.Pkg, 0, len(query.Needles))
if len(query.Needles) == 0 {
return found, nil
}

iterFound, errNeedle := a.gojqGetBatch(ctx, query)
if errNeedle != nil {
return nil, errNeedle
}

found = append(found, iterFound...)

return found, nil
}

// Get returns a list of packages that provide the given search term
func (a *AURCache) FindPackage(needle string) ([]*aur.Pkg, error) {
if pkgs, ok := a.provideCache[needle]; ok {
func (a *AURCache) FindPackage(ctx context.Context, needle string) ([]*aur.Pkg, error) {
cacheKey := a.cacheKey(needle, true, true, true)
if pkgs, ok := a.searchCache[cacheKey]; ok {
a.cacheHits++
return pkgs, nil
}

final, error := a.gojqGet(needle)
final, error := a.gojqGet(ctx, needle)
if error != nil {
return nil, error
}

a.provideCache[needle] = final
a.searchCache[cacheKey] = final

return final, nil
}

func (a *AURCache) gjsonGet(depName string) ([]*aur.Pkg, error) {
dedupMap := make(map[string]bool)
queryProvides := fmt.Sprintf("#(Provides.#(==\"%s\"))#", depName)
queryNames := fmt.Sprintf("#(Name==\"%s\")#", depName)
queryBases := fmt.Sprintf("#(PackageBase==\"%s\")#", depName)
func (a *AURCache) gojqGetBatch(ctx context.Context, query *AURQuery) ([]*aur.Pkg, error) {
pattern := ".[] | select("
for i, searchTerm := range query.Needles {
if i != 0 {
pattern += " or "
}

if query.ByName {
pattern += fmt.Sprintf("(.Name == \"%s\")", searchTerm)
if query.ByBase || query.ByProvides {
pattern += " or "
}
}

if query.ByBase {
pattern += fmt.Sprintf("(.PackageBase == \"%s\")", searchTerm)
if query.ByProvides {
pattern += " or "
}
}

if query.ByProvides {
pattern += fmt.Sprintf("(.Provides[]? == \"%s\")", searchTerm)
}
}

results := gjson.GetManyBytes(a.cache, queryProvides, queryNames, queryBases)
pattern += ")"

aggregated := append(append(results[0].Array(), results[1].Array()...), results[2].Array()...)
parsed, err := gojq.Parse(pattern)
if err != nil {
log.Fatalln(err)
}

final := make([]*aur.Pkg, 0, len(aggregated))
final := make([]*aur.Pkg, 0, len(query.Needles))

for i := range aggregated {
jsonString := aggregated[i].Raw
key := jsonString[:15]
iter := parsed.RunWithContext(ctx, a.unmarshalledCache) // or query.RunWithContext

if _, ok := dedupMap[key]; !ok {
pkg := &aur.Pkg{}
json.Unmarshal([]byte(jsonString), pkg)
final = append(final, pkg)
dedupMap[key] = true
for v, ok := iter.Next(); ok; v, ok = iter.Next() {
if err, ok := v.(error); ok {
return nil, err
}

pkg := new(aur.Pkg)
bValue, err := gojq.Marshal(v)
if err != nil {
log.Fatalln(err)
}

oj.Unmarshal(bValue, pkg)
final = append(final, pkg)
}

if a.DebugLoggerFn != nil {
a.DebugLoggerFn("AUR Query", pattern, "Found", len(final))
}

return final, nil
}

func (a *AURCache) gojqGet(searchTerm string) ([]*aur.Pkg, error) {
func (a *AURCache) gojqGet(ctx context.Context, searchTerm string) ([]*aur.Pkg, error) {
final := make([]*aur.Pkg, 0, 1)

iter := a.gojqCode.Run(a.unmarshalledCache, searchTerm) // or query.RunWithContext
for {
v, ok := iter.Next()
if !ok {
break
}
iter := a.gojqCode.RunWithContext(ctx, a.unmarshalledCache, searchTerm) // or query.RunWithContext

for v, ok := iter.Next(); ok; v, ok = iter.Next() {
if err, ok := v.(error); ok {
return nil, err
}
Expand Down
Loading

0 comments on commit 742b6ad

Please sign in to comment.