diff --git a/Dockerfile b/Dockerfile index d4c8978c9ae..93ba1010fc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,8 +24,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 CGO_CFLAGS="-D_LARGEFILE64_ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ - cscli hub update && \ - ./docker/preload-hub-items && \ + cscli hub update --with-content && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/Dockerfile.debian b/Dockerfile.debian index 73a44e8a4b8..dd55d2c3e48 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -29,8 +29,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ - cscli hub update && \ - ./docker/preload-hub-items && \ + cscli hub update --with-content && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index e6cba08940a..34bd61b1277 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -99,9 +99,10 @@ func (cli *cliHub) newListCmd() *cobra.Command { return cmd } -func (cli *cliHub) update(ctx context.Context) error { +func (cli *cliHub) update(ctx context.Context, withContent bool) error { local := cli.cfg().Hub remote := require.RemoteHub(ctx, cli.cfg()) + remote.EmbedItemContent = withContent // don't use require.Hub because if there is no index file, it would fail hub, err := cwhub.NewHub(local, remote, log.StandardLogger()) @@ -125,6 +126,8 @@ func (cli *cliHub) update(ctx context.Context) error { } func (cli *cliHub) newUpdateCmd() *cobra.Command { + withContent := false + cmd := &cobra.Command{ Use: "update", Short: "Download the latest index (catalog of available configurations)", @@ -134,10 +137,13 @@ Fetches the .index.json file from the hub, containing the list of available conf Args: cobra.ExactArgs(0), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { - return cli.update(cmd.Context()) + return cli.update(cmd.Context(), withContent) }, } + flags := cmd.Flags() + flags.BoolVar(&withContent, "with-content", false, "Download index with embedded item content") + return cmd } diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 0ae8841e029..fb87c1eff9b 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -57,7 +57,7 @@ run_hub_update() { index_modification_time=$(stat -c %Y /etc/crowdsec/hub/.index.json 2>/dev/null) # Run cscli hub update if no date or if the index file is older than 24h if [ -z "$index_modification_time" ] || [ $(( $(date +%s) - index_modification_time )) -gt 86400 ]; then - cscli hub update + cscli hub update --with-content else echo "Skipping hub update, index file is recent" fi diff --git a/pkg/cwhub/remote.go b/pkg/cwhub/remote.go index 04e4fab972f..8d2dc2dbb94 100644 --- a/pkg/cwhub/remote.go +++ b/pkg/cwhub/remote.go @@ -3,6 +3,7 @@ package cwhub import ( "context" "fmt" + "net/url" "github.com/sirupsen/logrus" @@ -11,9 +12,10 @@ import ( // RemoteHubCfg is used to retrieve index and items from the remote hub. type RemoteHubCfg struct { - Branch string - URLTemplate string - IndexPath string + Branch string + URLTemplate string + IndexPath string + EmbedItemContent bool } // urlTo builds the URL to download a file from the remote hub. @@ -30,6 +32,24 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) { return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil } +// addURLParam adds the "with_content=true" parameter to the URL if it's not already present. +func addURLParam(rawURL string, param string, value string) (string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + query := parsedURL.Query() + + if _, exists := query[param]; !exists { + query.Add(param, value) + } + + parsedURL.RawQuery = query.Encode() + + return parsedURL.String(), nil +} + // fetchIndex downloads the index from the hub and returns the content. func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, error) { if r == nil { @@ -41,6 +61,13 @@ func (r *RemoteHubCfg) fetchIndex(ctx context.Context, destPath string) (bool, e return false, fmt.Errorf("failed to build hub index request: %w", err) } + if r.EmbedItemContent { + url, err = addURLParam(url, "with_content", "true") + if err != nil { + return false, fmt.Errorf("failed to add 'with_content' parameter to URL: %w", err) + } + } + downloaded, err := downloader. New(). WithHTTPClient(hubClient). diff --git a/test/lib/config/config-local b/test/lib/config/config-local index f80c26bc652..3e3c806b616 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -114,7 +114,7 @@ make_init_data() { ./instance-db config-yaml ./instance-db setup - "$CSCLI" --warning hub update + "$CSCLI" --warning hub update --with-content # preload some content and data files "$CSCLI" collections install crowdsecurity/linux --download-only