From f6c5f8d28b352af80239b969276f9da6446dafb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 27 Feb 2023 12:17:44 +0100 Subject: [PATCH] Asasdf Fixes #10757 --- common/hexec/exec.go | 3 +- config/security/securityConfig.go | 21 ++++-- deps/deps.go | 67 +++++++++++++++++++ go.mod | 4 ++ go.sum | 4 ++ helpers/path.go | 24 +++++++ hugolib/hugo_sites_build.go | 2 + modules/config.go | 11 ++- .../resource_transformers/babel/babel.go | 5 +- .../tocss/dartsass/client.go | 7 +- .../tocss/dartsass/integration_test.go | 43 ++++++++++++ 11 files changed, 182 insertions(+), 9 deletions(-) diff --git a/common/hexec/exec.go b/common/hexec/exec.go index 7a9fdd938ae..02727de41d7 100644 --- a/common/hexec/exec.go +++ b/common/hexec/exec.go @@ -129,7 +129,8 @@ type Exec struct { // New will fail if name is not allowed according to the configured security policy. // Else a configured Runner will be returned ready to be Run. func (e *Exec) New(name string, arg ...any) (Runner, error) { - if err := e.sc.CheckAllowedExec(name); err != nil { + var err error + if name, err = e.sc.CheckAllowedExec(name); err != nil { return nil, err } diff --git a/config/security/securityConfig.go b/config/security/securityConfig.go index 4b0e0708606..1a940b5d375 100644 --- a/config/security/securityConfig.go +++ b/config/security/securityConfig.go @@ -40,9 +40,14 @@ var DefaultConfig = Config{ "^npx$", // used by all Node tools (Babel, PostCSS). "^postcss$", ), + AllowBinFromModules: NewWhitelist("^github.com/gohugoio/"), + // These have been tested to work with Hugo's external programs // on Windows, Linux and MacOS. OsEnv: NewWhitelist(`(?i)^((HTTPS?|NO)_PROXY|PATH(EXT)?|APPDATA|TE?MP|TERM|GO\w+)$`), + + // Maps the executable name to an absolute path. + ExecNameMap: map[string]string{}, }, Funcs: Funcs{ Getenv: NewWhitelist("^HUGO_", "^CI$"), @@ -70,8 +75,13 @@ type Config struct { // Exec holds os/exec policies. type Exec struct { - Allow Whitelist `json:"allow"` + Allow Whitelist `json:"allow"` + AllowBinFromModules Whitelist `json:"allowBinFromModules"` + OsEnv Whitelist `json:"osEnv"` + + // For internal use. + ExecNameMap map[string]string `json:"-"` } // Funcs holds template funcs policies. @@ -101,15 +111,18 @@ func (c Config) ToTOML() string { return strings.TrimSpace(b.String()) } -func (c Config) CheckAllowedExec(name string) error { +func (c Config) CheckAllowedExec(name string) (string, error) { if !c.Exec.Allow.Accept(name) { - return &AccessDeniedError{ + return "", &AccessDeniedError{ name: name, path: "security.exec.allow", policies: c.ToTOML(), } } - return nil + if path, found := c.Exec.ExecNameMap[name]; found { + return path, nil + } + return name, nil } diff --git a/deps/deps.go b/deps/deps.go index 6842c73310e..d90897199da 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -1,13 +1,17 @@ package deps import ( + "errors" "fmt" + "path" "path/filepath" + "runtime" "strings" "sync" "sync/atomic" "time" + "github.com/bep/githubreleasedownloader" "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" @@ -18,8 +22,10 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/postpub" + "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/metrics" "github.com/gohugoio/hugo/output" @@ -246,6 +252,67 @@ func New(cfg DepsCfg) (*Deps, error) { if err != nil { return nil, fmt.Errorf("failed to create security config from configuration: %w", err) } + + if cfg.Cfg.IsSet(("allModules")) { + allModules := cfg.Cfg.Get("allModules").(modules.Modules) + for _, m := range allModules { + mcfg := m.Config() + if mcfg.Type != modules.TypeBin { + continue + } + if !securityConfig.Exec.AllowBinFromModules.Accept(m.Path()) { + // TODO1 + return nil, fmt.Errorf("module %q is not allowed to install binaries", m.Path()) + } + fmt.Println("initSites", m.Path()) + + binDir, err := helpers.GetBinDir() + if err != nil { + return nil, err + } + if binDir == "" { + // TODO1 warning/error? + continue + } + + params := mcfg.Params + var ( + release githubreleasedownloader.Release + binName string + ) + if m, ok := params["github_release"]; ok { + if err := mapstructure.Decode(m, &release); err != nil { + return nil, err + } + } else { + return nil, errors.New("github_release not set") + } + if m, ok := params["bin"]; ok { + binName = m.(string) + } + if binName == "" { + return nil, errors.New("bin not set") + } + + // TOODO1 ... + goos := "macos" + goarch := strings.ToLower(runtime.GOARCH) + assetsFiltered := release.Assets.Filter(func(a githubreleasedownloader.Asset) bool { + return strings.Contains(a.Name, goos) && strings.Contains(a.Name, goarch) + }) + + if len(assetsFiltered) != 1 { + return nil, errors.New("no asset found") + } + + asset := assetsFiltered[0] + if err := githubreleasedownloader.DownloadAndExtractAssetToDir(asset, binDir); err != nil { + return nil, err + } + securityConfig.Exec.ExecNameMap[path.Base(binName)] = filepath.Join(binDir, binName) + } + } + execHelper := hexec.New(securityConfig) var filenameHasPostProcessPrefixMu sync.Mutex diff --git a/go.mod b/go.mod index 15be29127bd..16f185c60b8 100644 --- a/go.mod +++ b/go.mod @@ -104,6 +104,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect github.com/aws/smithy-go v1.13.4 // indirect + github.com/bep/githubreleasedownloader v0.0.0-20230227104846-7134298d0f32 // indirect + github.com/bep/workers v1.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -141,3 +143,5 @@ require ( ) go 1.18 + +replace github.com/bep/githubreleasedownloader => /Users/bep/dev/go/bep/githubreleasedownloader diff --git a/go.sum b/go.sum index 6c174422ccd..ae11bd4b6b1 100644 --- a/go.sum +++ b/go.sum @@ -583,6 +583,8 @@ github.com/bep/clock v0.3.0 h1:vfOA6+wVb6pPQEiXow9f/too92vNTLe9MuwO13PfI0M= github.com/bep/clock v0.3.0/go.mod h1:6Gz2lapnJ9vxpvPxQ2u6FcXFRoj4kkiqQ6pm0ERZlwk= github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= +github.com/bep/githubreleasedownloader v0.0.0-20230227104846-7134298d0f32 h1:dxHa79l69Gwe5aKX4N6sp0KFJ+vhekfB1Du4SbeE0z8= +github.com/bep/githubreleasedownloader v0.0.0-20230227104846-7134298d0f32/go.mod h1:M7i7Uo76ke0h473z5L6rFfDbEP+8XMElBzvPgviJUTM= github.com/bep/gitmap v1.1.2 h1:zk04w1qc1COTZPPYWDQHvns3y1afOsdRfraFQ3qI840= github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= github.com/bep/goat v0.5.0 h1:S8jLXHCVy/EHIoCY+btKkmcxcXFd34a0Q63/0D4TKeA= @@ -601,6 +603,8 @@ github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg= github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs= +github.com/bep/workers v1.1.0 h1:3Xw/1y/Fzjt8KBB4nCHfXcvyWH9h56iEwPquCjKphCc= +github.com/bep/workers v1.1.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= diff --git a/helpers/path.go b/helpers/path.go index 7bc216ec813..94079ce94bd 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -414,6 +414,30 @@ func GetCacheDir(fs afero.Fs, cfg config.Provider) (string, error) { return GetTempDir("hugo_cache", fs), nil } +// GetBinDir returns the directory where Hugo should store any binaries. +// Note that it's the user's responsibility to ensure that the directory is +// exists on the PATH. +func GetBinDir() (string, error) { + binDir := os.Getenv("HUGO_BIN") + if binDir == "" { + return "", nil + } + + binDir = addTrailingFileSeparator(binDir) + + exists, err := DirExists(binDir, hugofs.Os) + if err != nil { + return "", err + } + + if !exists { + return "", fmt.Errorf("HUGO_BIN env variable is set to no a directory that doesn't exist: %s", binDir) + } + + return binDir, nil + +} + func getCacheDir(cfg config.Provider) string { // Always use the cacheDir config if set. cacheDir := cfg.GetString("cacheDir") diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 66abf4f1655..92112322557 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -256,6 +256,8 @@ func (h *HugoSites) assemble(bcfg *BuildCfg) error { return nil } + { + } if err := h.getContentMaps().AssemblePages(); err != nil { return err } diff --git a/modules/config.go b/modules/config.go index 9d516e841cc..87fc39c9f54 100644 --- a/modules/config.go +++ b/modules/config.go @@ -27,9 +27,15 @@ import ( "github.com/mitchellh/mapstructure" ) -const WorkspaceDisabled = "off" +const ( + WorkspaceDisabled = "off" + + TypeBasic = "basic" + TypeBin = "bin" +) var DefaultModuleConfig = Config{ + Type: TypeBasic, // Default to direct, which means "git clone" and similar. We // will investigate proxy settings in more depth later. @@ -286,6 +292,9 @@ type Config struct { Mounts []Mount Imports []Import + // The module type, default is "basic" + Type string + // Meta info about this module (license information etc.). Params map[string]any diff --git a/resources/resource_transformers/babel/babel.go b/resources/resource_transformers/babel/babel.go index 9a9110f62bd..799e24fa69b 100644 --- a/resources/resource_transformers/babel/babel.go +++ b/resources/resource_transformers/babel/babel.go @@ -114,11 +114,12 @@ func (t *babelTransformation) Key() internal.ResourceTransformationKey { // npm install -g @babel/preset-env // Instead of installing globally, you can also install everything as a dev-dependency (--save-dev instead of -g) func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - const binaryName = "babel" + var binaryName = "babel" ex := t.rs.ExecHelper - if err := ex.Sec().CheckAllowedExec(binaryName); err != nil { + var err error + if binaryName, err = ex.Sec().CheckAllowedExec(binaryName); err != nil { return err } diff --git a/resources/resource_transformers/tocss/dartsass/client.go b/resources/resource_transformers/tocss/dartsass/client.go index f45e6537abd..4d3b5b86582 100644 --- a/resources/resource_transformers/tocss/dartsass/client.go +++ b/resources/resource_transformers/tocss/dartsass/client.go @@ -44,11 +44,16 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) return &Client{dartSassNotAvailable: true}, nil } - if err := rs.ExecHelper.Sec().CheckAllowedExec(dartSassEmbeddedBinaryName); err != nil { + var binaryName string + var err error + if binaryName, err = rs.ExecHelper.Sec().CheckAllowedExec(dartSassEmbeddedBinaryName); err != nil { return nil, err } + fmt.Println("binaryName", binaryName) + transpiler, err := godartsass.Start(godartsass.Options{ + DartSassEmbeddedFilename: binaryName, LogEventHandler: func(event godartsass.LogEvent) { message := strings.ReplaceAll(event.Message, dartSassStdinPrefix, "") switch event.Type { diff --git a/resources/resource_transformers/tocss/dartsass/integration_test.go b/resources/resource_transformers/tocss/dartsass/integration_test.go index aa71101a04e..7d2332102ca 100644 --- a/resources/resource_transformers/tocss/dartsass/integration_test.go +++ b/resources/resource_transformers/tocss/dartsass/integration_test.go @@ -14,6 +14,8 @@ package dartsass_test import ( + "fmt" + "os" "strings" "testing" @@ -498,3 +500,44 @@ T1: {{ $r.Content }} b.AssertLogMatches(`INFO.*Dart Sass: .*assets.*main.scss:14:0: number`) } + +func TestDartSassBinaryFromModule(t *testing.T) { + t.Parallel() + + //binDir := t.TempDir() + binDir := "/Users/bep/dev/go/bep/hugobin" + + //defer os.RemoveAll(binDir) + fmt.Println("Bin dir:", binDir) + os.Setenv("HUGO_BIN", binDir) + //existingPath := os.Getenv("PATH") + //os.Setenv("PATH", binDir+"/sass_embedded"+string(os.PathListSeparator)+existingPath) + + files := ` +-- hugo.toml -- +disableKinds = ['RSS','sitemap','taxonomy','term','page','section'] +[[module.mounts]] +source = 'assets' +target = 'assets' +[[module.imports]] +path="github.com/gohugoio/hugo-mod-bin-dartsass" +-- go.mod -- +module testdartsass +-- assets/main.scss -- +body {color: blue;} +-- layouts/index.html -- +{{- $opts := dict "transpiler" "dartsass" }} +{{- with resources.Get "main.scss" | toCSS $opts }}{{ .Content | safeHTML }}{{ end }} + + + ` + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + NeedsOsFS: true, + }, + ).Build() + + b.AssertFileContent("public/index.html", "body {\n color: blue;\n}") +}