diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index a44e76ae47d..dd98cd092cb 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -89,7 +89,6 @@ func HubDownloader(ctx context.Context, c *csconfig.Config) *cwhub.Downloader { remote := &cwhub.Downloader{ Branch: branch, URLTemplate: urlTemplate, - IndexPath: ".index.json", } return remote @@ -115,7 +114,7 @@ func Hub(c *csconfig.Config, logger *logrus.Logger) (*cwhub.Hub, error) { } if err := hub.Load(); err != nil { - return nil, fmt.Errorf("failed to read hub index: %w. Run 'sudo cscli hub update' to download the index again", err) + return nil, err } return hub, nil diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 94a1d6ef6fd..befd279ff65 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -29,10 +29,9 @@ const mockURLTemplate = "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s" var responseByPath map[string]string -// testHub initializes a temporary hub with an empty json file, optionally updating it. -func testHub(t *testing.T, update bool) *Hub { - tmpDir, err := os.MkdirTemp("", "testhub") - require.NoError(t, err) +// testHubOld initializes a temporary hub with an empty json file, optionally updating it. +func testHubOld(t *testing.T, update bool) *Hub { + tmpDir := t.TempDir() local := &csconfig.LocalHubCfg{ HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), @@ -41,7 +40,7 @@ func testHub(t *testing.T, update bool) *Hub { InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(local.HubDir, 0o700) + err := os.MkdirAll(local.HubDir, 0o700) require.NoError(t, err) err = os.MkdirAll(local.InstallDir, 0o700) @@ -53,10 +52,6 @@ func testHub(t *testing.T, update bool) *Hub { err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644) require.NoError(t, err) - t.Cleanup(func() { - os.RemoveAll(tmpDir) - }) - hub, err := NewHub(local, log.StandardLogger()) require.NoError(t, err) @@ -64,11 +59,10 @@ func testHub(t *testing.T, update bool) *Hub { indexProvider := &Downloader{ Branch: "master", URLTemplate: mockURLTemplate, - IndexPath: ".index.json", } ctx := context.Background() - err := hub.Update(ctx, indexProvider, false) + err = hub.Update(ctx, indexProvider, false) require.NoError(t, err) } @@ -92,7 +86,7 @@ func envSetup(t *testing.T) *Hub { // Mock the http client HubClient.Transport = newMockTransport() - hub := testHub(t, true) + hub := testHubOld(t, true) return hub } diff --git a/pkg/cwhub/doc.go b/pkg/cwhub/doc.go index b85d7634da4..fb7209b77ae 100644 --- a/pkg/cwhub/doc.go +++ b/pkg/cwhub/doc.go @@ -90,7 +90,6 @@ // indexProvider := cwhub.Downloader{ // URLTemplate: "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s", // Branch: "master", -// IndexPath: ".index.json", // } // // The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 48cb2382668..fa92e9960de 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,11 +12,13 @@ import ( "github.com/crowdsecurity/go-cs-lib/downloader" ) +// no need to import the lib package to use this +type NotFoundError = downloader.NotFoundError + // Downloader is used to retrieve index and items from a remote hub, with cache control. type Downloader struct { Branch string URLTemplate string - IndexPath string } // IndexProvider retrieves and writes .index.json @@ -61,7 +63,7 @@ func addURLParam(rawURL string, param string, value string) (string, error) { // It uses a temporary file to avoid partial downloads, and won't overwrite the original // if it has not changed. func (d *Downloader) FetchIndex(ctx context.Context, destPath string, withContent bool, logger *logrus.Logger) (bool, error) { - url, err := d.urlTo(d.IndexPath) + url, err := d.urlTo(".index.json") if err != nil { return false, fmt.Errorf("failed to build hub index request: %w", err) } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index fc0b257a284..7b0b99c28d8 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -10,43 +10,173 @@ import ( "testing" "github.com/sirupsen/logrus" + logtest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestFetchIndex(t *testing.T) { ctx := context.Background() mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + if r.URL.Query().Get("with_content") == "true" { - w.WriteHeader(http.StatusOK) _, err := w.Write([]byte(`Hi I'm an index with content`)) assert.NoError(t, err) } else { - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(`Hi I'm a regular index`)) + _, err := w.Write([]byte(`Hi I'm a minified index`)) assert.NoError(t, err) } })) defer mockServer.Close() + discard := logrus.New() + discard.Out = io.Discard + downloader := &Downloader{ - Branch: "main", URLTemplate: mockServer.URL + "/%s/%s", - IndexPath: "index.txt", } - logger := logrus.New() - logger.Out = io.Discard - - destPath := filepath.Join(t.TempDir(), "index.txt") + destPath := filepath.Join(t.TempDir(), "index-here") withContent := true - downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, logger) + var notFoundError NotFoundError + + // bad branch + + downloader.Branch = "dev" + + downloaded, err := downloader.FetchIndex(ctx, destPath, withContent, discard) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // ok + + downloader.Branch = "main" + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) require.NoError(t, err) assert.True(t, downloaded) content, err := os.ReadFile(destPath) require.NoError(t, err) assert.Equal(t, "Hi I'm an index with content", string(content)) + + // not "downloading" a second time + // since we don't have cache control in the mockServer, + // the file is downloaded to a temporary location but not replaced + + downloaded, err = downloader.FetchIndex(ctx, destPath, withContent, discard) + require.NoError(t, err) + assert.False(t, downloaded) + + // download without item content + + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err = os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, "Hi I'm a minified index", string(content)) + + // bad domain name + + downloader.URLTemplate = "x/%s/%s" + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + cstest.AssertErrorContains(t, err, `Get "x/main/.index.json": unsupported protocol scheme ""`) + assert.False(t, downloaded) + + downloader.URLTemplate = "http://x/%s/%s" + downloaded, err = downloader.FetchIndex(ctx, destPath, !withContent, discard) + // can be no such host, server misbehaving, etc + cstest.AssertErrorContains(t, err, `Get "http://x/main/.index.json": dial tcp: lookup x`) + assert.False(t, downloaded) +} + +func TestFetchContent(t *testing.T) { + ctx := context.Background() + + wantContent := "{'description':'linux'}" + wantHash := "e557cb9e1cb051bc3b6a695e4396c5f8e0eff4b7b0d2cc09f7684e1d52ea2224" + remotePath := "collections/crowdsecurity/linux.yaml" + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/"+remotePath { + w.WriteHeader(http.StatusNotFound) + } + + _, err := w.Write([]byte(wantContent)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + wantURL := mockServer.URL + "/main/collections/crowdsecurity/linux.yaml" + + // bad branch + + hubDownloader := &Downloader{ + URLTemplate: mockServer.URL + "/%s/%s", + } + + discard := logrus.New() + discard.Out = io.Discard + + destPath := filepath.Join(t.TempDir(), "content-here") + + var notFoundError NotFoundError + + // bad branch + + hubDownloader.Branch = "dev" + + downloaded, url, err := hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Empty(t, url) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // bad path + + hubDownloader.Branch = "main" + + downloaded, url, err = hubDownloader.FetchContent(ctx, "collections/linux.yaml", destPath, wantHash, discard) + assert.Empty(t, url) + require.ErrorAs(t, err, ¬FoundError) + assert.False(t, downloaded) + + // hash mismatch: the file is not reported as downloaded because it's not replaced + + capture, hook := logtest.NewNullLogger() + capture.SetLevel(logrus.WarnLevel) + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, "1234", capture) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.False(t, downloaded) + cstest.RequireLogContains(t, hook, "hash mismatch: expected 1234, got "+wantHash) + + // ok + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.True(t, downloaded) + + content, err := os.ReadFile(destPath) + require.NoError(t, err) + assert.Equal(t, wantContent, string(content)) + + // not "downloading" a second time + // since we don't have cache control in the mockServer, + // the file is downloaded to a temporary location but not replaced + + downloaded, url, err = hubDownloader.FetchContent(ctx, remotePath, destPath, wantHash, discard) + assert.Equal(t, wantURL, url) + require.NoError(t, err) + assert.False(t, downloaded) } diff --git a/pkg/cwhub/fetch.go b/pkg/cwhub/fetch.go index dd1a520d7e2..e8dacad4a6d 100644 --- a/pkg/cwhub/fetch.go +++ b/pkg/cwhub/fetch.go @@ -11,8 +11,8 @@ import ( ) // writeEmbeddedContentTo writes the embedded content to the specified path and checks the hash. -// If the content is base64 encoded, it will be decoded before writing. Check for item.Content -// before calling this method. +// If the content is base64 encoded, it will be decoded before writing. Call this method only +// if item.Content if not empty. func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { if i.Content == "" { return fmt.Errorf("no embedded content for %s", i.Name) @@ -48,7 +48,9 @@ func (i *Item) writeEmbeddedContentTo(destPath, wantHash string) error { } // FetchContentTo writes the last version of the item's YAML file to the specified path. -// Returns whether the file was downloaded, and the remote url for feedback purposes. +// If the file is embedded in the index file, it will be written directly without downloads. +// Returns whether the file was downloaded (to inform if the security engine needs reloading) +// and the remote url for feedback purposes. func (i *Item) FetchContentTo(ctx context.Context, contentProvider ContentProvider, destPath string) (bool, string, error) { wantHash := i.latestHash() if wantHash == "" { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 3722ceaafcd..998a4032359 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -36,7 +36,7 @@ func (h *Hub) GetDataDir() string { // and check for unmanaged items. func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) { if local == nil { - return nil, errors.New("no hub configuration found") + return nil, errors.New("no hub configuration provided") } if logger == nil { @@ -58,14 +58,10 @@ func (h *Hub) Load() error { h.logger.Debugf("loading hub idx %s", h.local.HubIndexFile) if err := h.parseIndex(); err != nil { - return err - } - - if err := h.localSync(); err != nil { - return fmt.Errorf("failed to sync hub items: %w", err) + return fmt.Errorf("invalid hub index: %w. Run 'sudo cscli hub update' to download the index again", err) } - return nil + return h.localSync() } // parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections. @@ -153,12 +149,14 @@ func (h *Hub) ItemStats() []string { return ret } +var ErrUpdateAfterSync = errors.New("cannot update hub index after load/sync") + // Update downloads the latest version of the index and writes it to disk if it changed. -// It cannot be called after Load() unless the hub is completely empty. +// It cannot be called after Load() unless the index was completely empty. func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error { - if len(h.pathIndex) > 0 { + if len(h.items) > 0 { // if this happens, it's a bug. - return errors.New("cannot update hub after items have been loaded") + return ErrUpdateAfterSync } downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index c2b949b7cdf..461b59de78b 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -2,91 +2,261 @@ package cwhub import ( "context" - "fmt" + "net/http" + "net/http/httptest" "os" + "path/filepath" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func TestInitHubUpdate(t *testing.T) { - hub := envSetup(t) +// testHubCfg creates an empty hub structure in a temporary directory +// and returns its configuration object. +// +// This allow the reuse of the hub content for multiple instances +// of the Hub object. +func testHubCfg(t *testing.T) *csconfig.LocalHubCfg { + tempDir := t.TempDir() + + local := csconfig.LocalHubCfg{ + HubDir: filepath.Join(tempDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tempDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tempDir, "crowdsec"), + InstallDataDir: filepath.Join(tempDir, "installed-data"), + } + + err := os.MkdirAll(local.HubDir, 0o755) + require.NoError(t, err) - _, err := NewHub(hub.local, nil) + err = os.MkdirAll(local.InstallDir, 0o755) require.NoError(t, err) - ctx := context.Background() + err = os.MkdirAll(local.InstallDataDir, 0o755) + require.NoError(t, err) + + return &local +} - indexProvider := &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", +func testHub(t *testing.T, localCfg *csconfig.LocalHubCfg, indexJson string) (*Hub, error) { + if localCfg == nil { + localCfg = testHubCfg(t) } - err = hub.Update(ctx, indexProvider, false) + err := os.WriteFile(localCfg.HubIndexFile, []byte(indexJson), 0o644) require.NoError(t, err) + hub, err := NewHub(localCfg, nil) + require.NoError(t, err) err = hub.Load() + + return hub, err +} + +func TestIndexEmpty(t *testing.T) { + // an empty hub is valid, and should not have warnings + hub, err := testHub(t, nil, "{}") require.NoError(t, err) + assert.Empty(t, hub.Warnings) } -func TestUpdateIndex(t *testing.T) { - // bad url template - fmt.Println("Test 'bad URL'") +func TestIndexJSON(t *testing.T) { + // but it can't be an empty string + hub, err := testHub(t, nil, "") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) + + // it must be valid json + hub, err = testHub(t, nil, "def not json") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: invalid character 'd' looking for beginning of value. Run 'sudo cscli hub update' to download the index again") + assert.Empty(t, hub.Warnings) + + hub, err = testHub(t, nil, "{") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) - tmpIndex, err := os.CreateTemp("", "index.json") + // and by json we mean an object + hub, err = testHub(t, nil, "[]") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: json: cannot unmarshal array into Go value of type cwhub.HubItems") + assert.Empty(t, hub.Warnings) +} + +func TestIndexUnknownItemType(t *testing.T) { + // Allow unknown fields in the top level object, likely new item types + hub, err := testHub(t, nil, `{"goodies": {}}`) require.NoError(t, err) + assert.Empty(t, hub.Warnings) +} - // close the file to avoid preventing the rename on windows - err = tmpIndex.Close() +func TestHubUpdate(t *testing.T) { + // update an empty hub with a index containing a parser. + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - t.Cleanup(func() { - os.Remove(tmpIndex.Name()) - }) + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` - hub := envSetup(t) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } - hub.local.HubIndexFile = tmpIndex.Name() + _, err = w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() ctx := context.Background() - indexProvider := &Downloader{ - URLTemplate: "x", - Branch: "", - IndexPath: "", + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") + err = hub.Update(ctx, downloader, true) + require.NoError(t, err) + + err = hub.Load() + require.NoError(t, err) + + item := hub.GetItem("parsers", "author/pars1") + assert.NotEmpty(t, item) + assert.Equal(t, "author/pars1", item.Name) +} + +func TestHubUpdateInvalidTemplate(t *testing.T) { + hub, err := testHub(t, nil, "{}") + require.NoError(t, err) - // bad domain - fmt.Println("Test 'bad domain'") + ctx := context.Background() - indexProvider = &Downloader{ - URLTemplate: "https://baddomain/crowdsecurity/%s/%s", - Branch: "master", - IndexPath: ".index.json", + downloader := &Downloader{ + Branch: "main", + URLTemplate: "x", } - err = hub.Update(ctx, indexProvider, false) + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorMessage(t, err, "failed to build hub index request: invalid URL template 'x'") +} + +func TestHubUpdateCannotWrite(t *testing.T) { + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - // XXX: this is not failing - // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") - // bad target path - fmt.Println("Test 'bad target path'") + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + + _, err = w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() - indexProvider = &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", - IndexPath: ".index.json", + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", } - hub.local.HubIndexFile = "/does/not/exist/index.json" + hub.local.HubIndexFile = "/proc/foo/bar/baz/.index.json" + + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorContains(t, err, "failed to create temporary download file for /proc/foo/bar/baz/.index.json") +} + +func TestHubUpdateAfterLoad(t *testing.T) { + // Update() can't be called after Load() if the hub is not completely empty. + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + hub, err := testHub(t, nil, index1) + require.NoError(t, err) + + index2 := ` +{ + "parsers": { + "author/pars2": { + "path": "parsers/s01-parse/pars2.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + + _, err = w.Write([]byte(index2)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") + err = hub.Update(ctx, downloader, true) + require.ErrorIs(t, err, ErrUpdateAfterSync) } diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 74b1cfa3ebe..38385d9399d 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -11,8 +11,6 @@ import ( "github.com/Masterminds/semver/v3" yaml "gopkg.in/yaml.v3" - - "github.com/crowdsecurity/crowdsec/pkg/emoji" ) const ( @@ -46,62 +44,6 @@ type ItemVersion struct { Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` } -// ItemState is used to keep the local state (i.e. at runtime) of an item. -// This data is not stored in the index, but is displayed with "cscli ... inspect". -type ItemState struct { - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` - LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` - Installed bool `json:"installed"` - Downloaded bool `json:"downloaded"` - UpToDate bool `json:"up_to_date"` - Tainted bool `json:"tainted"` - TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` -} - -// IsLocal returns true if the item has been create by a user (not downloaded from the hub). -func (s *ItemState) IsLocal() bool { - return s.Installed && !s.Downloaded -} - -// Text returns the status of the item as a string (eg. "enabled,update-available"). -func (s *ItemState) Text() string { - ret := "disabled" - - if s.Installed { - ret = "enabled" - } - - if s.IsLocal() { - ret += ",local" - } - - if s.Tainted { - ret += ",tainted" - } else if !s.UpToDate && !s.IsLocal() { - ret += ",update-available" - } - - return ret -} - -// Emoji returns the status of the item as an emoji (eg. emoji.Warning). -func (s *ItemState) Emoji() string { - switch { - case s.IsLocal(): - return emoji.House - case !s.Installed: - return emoji.Prohibited - case s.Tainted || (!s.UpToDate && !s.IsLocal()): - return emoji.Warning - case s.Installed: - return emoji.CheckMark - default: - return emoji.QuestionMark - } -} - type Dependencies struct { Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` @@ -292,49 +234,11 @@ func (i *Item) CurrentDependencies() Dependencies { } func (i *Item) logMissingSubItems() { - if !i.HasSubItems() { - return - } - - for _, subName := range i.Parsers { - if i.hub.GetItem(PARSERS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name) - } - } - - for _, subName := range i.Scenarios { - if i.hub.GetItem(SCENARIOS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name) - } - } - - for _, subName := range i.PostOverflows { - if i.hub.GetItem(POSTOVERFLOWS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name) - } - } - - for _, subName := range i.Contexts { - if i.hub.GetItem(CONTEXTS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name) - } - } - - for _, subName := range i.AppsecConfigs { - if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) - } - } - - for _, subName := range i.AppsecRules { - if i.hub.GetItem(APPSEC_RULES, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name) - } - } - - for _, subName := range i.Collections { - if i.hub.GetItem(COLLECTIONS, subName) == nil { - i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) + for _, sub := range i.CurrentDependencies().byType() { + for _, subName := range sub.itemNames { + if i.hub.GetItem(sub.typeName, subName) == nil { + i.hub.logger.Errorf("can't find %s:%s, required by %s", sub.typeName, subName, i.Name) + } } } } diff --git a/pkg/cwhub/item_test.go b/pkg/cwhub/item_test.go index 703bbb5cb90..350861ff85e 100644 --- a/pkg/cwhub/item_test.go +++ b/pkg/cwhub/item_test.go @@ -6,39 +6,16 @@ import ( "github.com/stretchr/testify/require" ) -func TestItemStatus(t *testing.T) { +func TestItemStats(t *testing.T) { hub := envSetup(t) // get existing map x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) - // Get item: good and bad - for k := range x { - item := hub.GetItem(COLLECTIONS, k) - require.NotNil(t, item) - - item.State.Installed = true - item.State.UpToDate = false - item.State.Tainted = false - item.State.Downloaded = true - - txt := item.State.Text() - require.Equal(t, "enabled,update-available", txt) - - item.State.Installed = true - item.State.UpToDate = false - item.State.Tainted = false - item.State.Downloaded = false - - txt = item.State.Text() - require.Equal(t, "enabled,local", txt) - } - stats := hub.ItemStats() require.Equal(t, []string{ "Loaded: 2 parsers, 1 scenarios, 3 collections", - "Unmanaged items: 3 local, 0 tainted", }, stats) } diff --git a/pkg/cwhub/itemupgrade_test.go b/pkg/cwhub/itemupgrade_test.go index da02837e972..3225d2f013b 100644 --- a/pkg/cwhub/itemupgrade_test.go +++ b/pkg/cwhub/itemupgrade_test.go @@ -41,7 +41,6 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub, err := NewHub(hub.local, remote, nil) @@ -101,7 +100,6 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) @@ -173,7 +171,6 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te remote := &Downloader{ URLTemplate: mockURLTemplate, Branch: "master", - IndexPath: ".index.json", } hub = getHubOrFail(t, hub.local, remote) diff --git a/pkg/cwhub/state.go b/pkg/cwhub/state.go new file mode 100644 index 00000000000..518185aff1c --- /dev/null +++ b/pkg/cwhub/state.go @@ -0,0 +1,61 @@ +package cwhub + +import ( + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +// ItemState is used to keep the local state (i.e. at runtime) of an item. +// This data is not stored in the index, but is displayed with "cscli ... inspect". +type ItemState struct { + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` +} + +// IsLocal returns true if the item has been create by a user (not downloaded from the hub). +func (s *ItemState) IsLocal() bool { + return s.Installed && !s.Downloaded +} + +// Text returns the status of the item as a string (eg. "enabled,update-available"). +func (s *ItemState) Text() string { + ret := "disabled" + + if s.Installed { + ret = "enabled" + } + + if s.IsLocal() { + ret += ",local" + } + + if s.Tainted { + ret += ",tainted" + } else if !s.UpToDate && !s.IsLocal() { + ret += ",update-available" + } + + return ret +} + +// Emoji returns the status of the item as an emoji (eg. emoji.Warning). +func (s *ItemState) Emoji() string { + switch { + case s.IsLocal(): + return emoji.House + case !s.Installed: + return emoji.Prohibited + case s.Tainted || (!s.UpToDate && !s.IsLocal()): + return emoji.Warning + case s.Installed: + return emoji.CheckMark + default: + return emoji.QuestionMark + } +} diff --git a/pkg/cwhub/state_test.go b/pkg/cwhub/state_test.go new file mode 100644 index 00000000000..3ed3de16fcc --- /dev/null +++ b/pkg/cwhub/state_test.go @@ -0,0 +1,76 @@ +package cwhub + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/crowdsecurity/crowdsec/pkg/emoji" +) + +func TestItemStateText(t *testing.T) { + // Test the text representation of an item state. + type test struct { + state ItemState + want string + wantIcon string + } + + tests := []test{ + { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: false, + Downloaded: true, + }, + "enabled,update-available", + emoji.Warning, + }, { + ItemState{ + Installed: true, + UpToDate: true, + Tainted: false, + Downloaded: true, + }, + "enabled", + emoji.CheckMark, + }, { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: false, + Downloaded: false, + }, + "enabled,local", + emoji.House, + }, { + ItemState{ + Installed: false, + UpToDate: false, + Tainted: false, + Downloaded: true, + }, + "disabled,update-available", + emoji.Prohibited, + }, { + ItemState{ + Installed: true, + UpToDate: false, + Tainted: true, + Downloaded: true, + }, + "enabled,tainted", + emoji.Warning, + }, + } + + for idx, tc := range tests { + t.Run("Test "+strconv.Itoa(idx), func(t *testing.T) { + got := tc.state.Text() + assert.Equal(t, tc.want, got) + assert.Equal(t, tc.wantIcon, tc.state.Emoji()) + }) + } +} diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index d2b59df35d6..ee8e49f2bf0 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -108,6 +108,7 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo if len(subsHub) < 4 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subsHub)) } + stage = subsHub[1] fauthor = subsHub[2] fname = subsHub[3] @@ -167,6 +168,7 @@ func sortedVersions(raw []string) ([]string, error) { for idx, r := range raw { v, err := semver.NewVersion(r) if err != nil { + // TODO: should catch this during index parsing return nil, fmt.Errorf("%s: %w", r, err) } @@ -461,13 +463,12 @@ func removeDuplicates(sl []string) []string { // localSync updates the hub state with downloaded, installed and local items. func (h *Hub) localSync() error { - err := h.syncDir(h.local.InstallDir) - if err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) + if err := h.syncDir(h.local.InstallDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.InstallDir, err) } - if err = h.syncDir(h.local.HubDir); err != nil { - return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) + if err := h.syncDir(h.local.HubDir); err != nil { + return fmt.Errorf("failed to sync %s: %w", h.local.HubDir, err) } warnings := make([]string, 0) diff --git a/pkg/hubops/doc.go b/pkg/hubops/doc.go new file mode 100644 index 00000000000..b87a42653bc --- /dev/null +++ b/pkg/hubops/doc.go @@ -0,0 +1,45 @@ +/* +Package hubops is responsible for managing the local hub (items and data files) for CrowdSec. + +The index file itself (.index.json) is still managed by pkg/cwhub, which also provides the Hub +and Item structs. + +The hubops package is mostly used by cscli for the "cscli install/remove/upgrade ..." commands. + +It adopts a command-based pattern: a Plan contains a sequence of Commands. Both Plan and Command +have separate preparation and execution methods. + + - Command Interface: + The Command interface defines the contract for all operations that can be + performed on hub items. Each operation implements the Prepare and Run + methods, allowing for pre-execution setup and actual execution logic. + + - ActionPlan: + ActionPlan serves as a container for a sequence of Commands. It manages the + addition of commands, handles dependencies between them, and orchestrates their + execution. ActionPlan also provides a mechanism for interactive confirmation and dry-run. + +To perform operations on hub items, create an ActionPlan and add the desired +Commands to it. Once all commands are added, execute the ActionPlan to perform +the operations in the correct order, handling dependencies and user confirmations. + +Example: + + hub := cwhub.NewHub(...) + plan := hubops.NewActionPlan(hub) + + downloadCmd := hubops.NewDownloadCommand(item, force) + if err := plan.AddCommand(downloadCmd); err != nil { + logrus.Fatalf("Failed to add download command: %v", err) + } + + enableCmd := hubops.NewEnableCommand(item, force) + if err := plan.AddCommand(enableCmd); err != nil { + logrus.Fatalf("Failed to add enable command: %v", err) + } + + if err := plan.Execute(ctx, confirm, dryRun, verbose); err != nil { + logrus.Fatalf("Failed to execute action plan: %v", err) + } +*/ +package hubops diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index 03723ecc82b..b03b58732fa 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -82,8 +82,8 @@ teardown() { new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")') echo "$new_hub" >"$INDEX_PATH" rune -0 cscli hub list --error - assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb" - assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql" + assert_stderr --partial "can't find parsers:crowdsecurity/smb-logs, required by crowdsecurity/smb" + assert_stderr --partial "can't find scenarios:crowdsecurity/mysql-bf, required by crowdsecurity/mysql" } @test "loading hub reports tainted items (subitem is tainted)" { diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index 2f1c952848b..8ebe505c6e1 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -80,8 +80,8 @@ teardown() { rune -0 cscli collections install crowdsecurity/sshd rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics - # XXX: we are on the verbose side here... - assert_stderr "Error: failed to read hub index: failed to sync hub items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again" + # XXX: this must be triggered during parse, not sync + assert_stderr "Error: failed to sync $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version" } @test "removing or purging an item already removed by hand" { diff --git a/test/bats/hub-index.bats b/test/bats/hub-index.bats index 76759991e4a..a609974d67a 100644 --- a/test/bats/hub-index.bats +++ b/test/bats/hub-index.bats @@ -32,7 +32,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no index metadata." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no index metadata." } @test "malformed index - no download path" { @@ -46,7 +46,7 @@ teardown() { EOF rune -1 cscli hub list - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no download path." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no download path." } @test "malformed parser - no stage" { @@ -63,7 +63,7 @@ teardown() { EOF rune -1 cscli hub list -o raw - assert_stderr --partial "failed to read hub index: parsers:author/pars1 has no stage." + assert_stderr --partial "invalid hub index: parsers:author/pars1 has no stage." } @test "malformed parser - short path" {