Skip to content

Commit

Permalink
cmd/pkgsite: add multi-module support, invalidation, and search
Browse files Browse the repository at this point in the history 
This change adds a new goPackagesModuleGetter, which uses
x/tools/go/packages to query package information for modules requested
by the pkgsite command.

Additionally, add new extension interfaces that allow getters to add
support for search and content invalidation. The go/packages getter uses
these extensions to implement search (via a simple fuzzy-matching
algorithm copied from x/tools), and invalidation via statting package
files.

Along the way, refactor slightly for testing ergonomics.

Updates golang/go#40371
Updates golang/go#50229
Fixes golang/go#54479

Change-Id: Iea91a4d6327707733cbbc4f74a9d93052f33e295
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/474295
TryBot-Result: kokoro <noreply+kokoro@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
  • Loading branch information
@findleyr
findleyr committed Mar 14, 2023
1 parent 8e09d06 commit b5fdc90
Show file tree
Hide file tree
Showing 12 changed files with 1,200 additions and 435 deletions.
311 changes: 172 additions & 139 deletions cmd/pkgsite/main.go
View file

Large diffs are not rendered by default.

127 changes: 63 additions & 64 deletions cmd/pkgsite/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ import (

"github.com/google/go-cmp/cmp"
"golang.org/x/net/html"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/proxy/proxytest"
"golang.org/x/pkgsite/internal/testing/htmlcheck"
"golang.org/x/pkgsite/internal/testing/testhelper"
)

var (
Expand Down Expand Up @@ -51,48 +51,45 @@ func TestBuildGetters(t *testing.T) {
prox, teardown := proxytest.SetupTestClient(t, testModules)
defer teardown()

localGetter := "Dir(example.com/testmod, " + abs(localModule) + ")"
localGetter := "Dir(" + abs(localModule) + ")"
cacheGetter := "FSProxy(" + abs(cacheDir) + ")"
for _, test := range []struct {
name string
paths []string
cmods []internal.Modver
dirs []string
cacheDir string
prox *proxy.Client
proxy *proxy.Client
want []string
}{
{
name: "local only",
paths: []string{localModule},
want: []string{localGetter},
name: "local only",
dirs: []string{localModule},
want: []string{localGetter},
},
{
name: "cache",
cacheDir: cacheDir,
want: []string{cacheGetter},
},
{
name: "proxy",
prox: prox,
want: []string{"Proxy"},
name: "proxy",
proxy: prox,
want: []string{"Proxy"},
},
{
name: "all three",
paths: []string{localModule},
dirs: []string{localModule},
cacheDir: cacheDir,
prox: prox,
proxy: prox,
want: []string{localGetter, cacheGetter, "Proxy"},
},
{
name: "list",
paths: []string{localModule},
cacheDir: cacheDir,
cmods: []internal.Modver{{Path: "foo", Version: "v1.2.3"}},
want: []string{localGetter, "FSProxy(" + abs(cacheDir) + ", foo@v1.2.3)"},
},
} {
t.Run(test.name, func(t *testing.T) {
getters, err := buildGetters(ctx, test.paths, false, test.cacheDir, test.cmods, test.prox)
getters, err := buildGetters(ctx, getterConfig{
dirs: test.dirs,
pattern: "./...",
modCacheDir: test.cacheDir,
proxy: test.proxy,
})
if err != nil {
t.Fatal(err)
}
Expand All @@ -119,56 +116,91 @@ func TestServer(t *testing.T) {
return a
}

localModule := repoPath("internal/fetch/testdata/has_go_mod")
localModule, _ := testhelper.WriteTxtarToTempDir(t, `
-- go.mod --
module example.com/testmod
-- a.go --
package a
`)
cacheDir := repoPath("internal/fetch/testdata/modcache")
testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
prox, teardown := proxytest.SetupTestClient(t, testModules)
defer teardown()

getters, err := buildGetters(context.Background(), []string{localModule}, false, cacheDir, nil, prox)
if err != nil {
t.Fatal(err)
}
server, err := newServer(getters, prox)
if err != nil {
t.Fatal(err)
defaultConfig := serverConfig{
paths: []string{localModule},
gopathMode: false,
useListedMods: true,
useCache: true,
cacheDir: cacheDir,
proxy: prox,
}
mux := http.NewServeMux()
server.Install(mux.Handle, nil, nil)

modcacheChecker := in("",
in(".Documentation", hasText("var V = 1")),
sourceLinks(path.Join(abs(cacheDir), "modcache.com@v1.0.0"), "a.go"))

ctx := context.Background()
for _, test := range []struct {
name string
cfg serverConfig
url string
want htmlcheck.Checker
}{
{
"local",
defaultConfig,
"example.com/testmod",
in("",
in(".Documentation", hasText("There is no documentation for this package.")),
sourceLinks(path.Join(abs(localModule), "example.com/testmod"), "a.go")),
},
{
"modcache",
defaultConfig,
"modcache.com@v1.0.0",
modcacheChecker,
},
{
"modcache latest",
defaultConfig,
"modcache.com",
modcacheChecker,
},
{
"proxy",
defaultConfig,
"example.com/single/pkg",
hasText("G is new in v1.1.0"),
},
{
"search",
defaultConfig,
"search?q=a",
in(".SearchResults",
hasText("example.com/testmod"),
),
},
{
"search",
defaultConfig,
"search?q=zzz",
in(".SearchResults",
hasText("no matches"),
),
},
// TODO(rfindley): add more tests, including a test for the standard
// library once it doesn't go through the stdlib package.
// See also golang/go#58923.
} {
t.Run(test.name, func(t *testing.T) {
server, err := buildServer(ctx, test.cfg)
if err != nil {
t.Fatal(err)
}
mux := http.NewServeMux()
server.Install(mux.Handle, nil, nil)

w := httptest.NewRecorder()
mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil))
if w.Code != http.StatusOK {
Expand Down Expand Up @@ -203,36 +235,3 @@ func TestCollectPaths(t *testing.T) {
t.Errorf("got %v, want %v", got, want)
}
}

func TestListModsForPaths(t *testing.T) {
listModules = func(string) ([]listedMod, error) {
return []listedMod{
{
internal.Modver{Path: "m1", Version: "v1.2.3"},
"/dir/cache/download/m1/@v/v1.2.3.mod",
false,
},
{
internal.Modver{Path: "m2", Version: "v1.0.0"},
"/repos/m2/go.mod",
false,
},
{
internal.Modver{Path: "indir", Version: "v2.3.4"},
"",
true,
},
}, nil
}
defer func() { listModules = _listModules }()

gotPaths, gotCacheMods, err := listModsForPaths([]string{"m1"}, "/dir")
if err != nil {
t.Fatal(err)
}
wantPaths := []string{"/repos/m2"}
wantCacheMods := []internal.Modver{{Path: "m1", Version: "v1.2.3"}}
if !cmp.Equal(gotPaths, wantPaths) || !cmp.Equal(gotCacheMods, wantCacheMods) {
t.Errorf("got\n%v, %v\nwant\n%v, %v", gotPaths, gotCacheMods, wantPaths, wantCacheMods)
}
}
74 changes: 72 additions & 2 deletions internal/datasource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,73 @@

package internal

import "context"
import (
"context"
"time"
)

// SearchOptions provide information used by db.Search.
type SearchOptions struct {
// Maximum number of results to return (page size).
MaxResults int

// Offset for DB query.
Offset int

// Maximum number to use for total result count.
MaxResultCount int

// If true, perform a symbol search.
SearchSymbols bool

// SymbolFilter is the word in a search query with a # prefix.
SymbolFilter string
}

// SearchResult represents a single search result from SearchDocuments.
type SearchResult struct {
Name string
PackagePath string
ModulePath string
Version string
Synopsis string
Licenses []string

CommitTime time.Time

// Score is used to sort items in an array of SearchResult.
Score float64

// NumImportedBy is the number of packages that import PackagePath.
NumImportedBy uint64

// SameModule is a list of SearchResults from the same module as this one,
// with lower scores.
SameModule []*SearchResult

// OtherMajor is a map from module paths with the same series path but at
// different major versions of this module, to major version.
// The major version for a non-vN module path (either 0 or 1) is computed
// based on the version in search documents.
OtherMajor map[string]int

// NumResults is the total number of packages that were returned for this
// search.
NumResults uint64

// Symbol information returned by a search request.
// Only populated for symbol search mode.
SymbolName string
SymbolKind SymbolKind
SymbolSynopsis string
SymbolGOOS string
SymbolGOARCH string

// Offset is the 0-based number of this row in the DB query results, which
// is the value to use in a SQL OFFSET clause to have this row be the first
// one returned.
Offset int
}

// DataSource is the interface used by the frontend to interact with module data.
type DataSource interface {
Expand All @@ -22,10 +88,14 @@ type DataSource interface {
GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *UnitMeta, err error)
// GetModuleReadme gets the readme for the module.
GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*Readme, error)

// GetLatestInfo gets information about the latest versions of a unit and module.
// See LatestInfo for documentation.
GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *UnitMeta) (LatestInfo, error)

// SupportsSearch reports whether this data source supports search.
SupportsSearch() bool
// Search searches for packages matching the given query.
Search(ctx context.Context, q string, opts SearchOptions) (_ []*SearchResult, err error)
}

// LatestInfo holds information about the latest versions and paths.
Expand Down
Loading

0 comments on commit b5fdc90

Please sign in to comment.