From 7380696e4cb771d99db1f455f82e00290938f681 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 10:46:55 -0300 Subject: [PATCH 1/9] draft --- cmd/embedded.go | 168 ++++++++++++++++++++++++++++++++++++ main.go | 1 + modules/options/static.go | 31 +++++++ modules/public/static.go | 33 +++++++ modules/templates/static.go | 13 +++ 5 files changed, 246 insertions(+) create mode 100644 cmd/embedded.go diff --git a/cmd/embedded.go b/cmd/embedded.go new file mode 100644 index 0000000000000..41aab519d3df1 --- /dev/null +++ b/cmd/embedded.go @@ -0,0 +1,168 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build bindata + +package cmd + +import ( + "fmt" +// "os" +// "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/public" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" + + "github.com/gobwas/glob" + "github.com/urfave/cli" +) + +// Cmdembedded represents the available extract sub-command. +var ( + Cmdembedded = cli.Command{ + Name: "embedded", + Usage: "Extract embedded resources", + Description: "A command for extracting embedded resources, like templates and images.", + Subcommands: []cli.Command{ + subcmdList, + subcmdExtract, + }, + Flags: []cli.Flag{ + /* + cli.StringFlag{ + Name: "name, n", + Value: "**", + Usage: "glob pattern used to match files", + }, + */ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, + }, + } + + subcmdList = cli.Command{ + Name: "list", + Usage: "List files matching the given pattern", + Action: runList, + } + + subcmdExtract = cli.Command{ + Name: "extract", + Usage: "Extract resources", + Action: runExtract, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "overwrite", + Usage: "Overwrite files if they already exist", + }, + cli.BoolFlag{ + Name: "custom", + Usage: "Extract to the 'custom' directory", + }, + cli.StringFlag{ + Name: "destination", + Usage: "Destination for the extracted files", + }, + }, + } + + sections map[string]*section + assets []asset +) + +type section struct { + Path string + Names func() []string + IsDir func(string) (bool, error) +} + +type asset struct { + Section *section + Name string +} + +func initEmbeddedExtractor(c *cli.Context) error { + + // Silence the console logger + log.DelNamedLogger("console") + + // Read configuration file + setting.NewContext() + + pats, err := getPatterns(c.Args()) + if err != nil { + return err + } + sections := make(map[string]*section,3) + + sections["public"] = §ion{ Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir } + sections["options"] = §ion{ Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir } + sections["templates"] = §ion{ Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir } + + for _, sec := range sections { + assets = append(assets, buildAssetList(sec, pats, c)...) + } + + return nil +} + +func runList(ctx *cli.Context) error { + if err := initEmbeddedExtractor(ctx); err != nil { + return err + } + // fmt.Println("Using app.ini at", setting.CustomConf) + + for _, asset := range assets { + fmt.Printf("- [%s] [%s]\n", asset.Section.Path, asset.Name) + } + fmt.Println("End of list.") + return nil +} + +func runExtract(ctx *cli.Context) error { + fmt.Println("Not implemented") + return nil +} + +func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { + var results = make([]asset, 0, 64) + for _, name := range sec.Names() { + if isdir, err := sec.IsDir(name); !isdir && err == nil { + if sec.Path == "public" && + strings.HasPrefix(name, "vendor/") && + !c.Bool("include-vendored") { + continue + } + matchName := "/"+sec.Path+"/"+name + for _, g := range globs { + if g.Match(matchName) { + results = append(results, asset{Section: sec, Name: name}) + break + } + } + } + } + return results +} + +func getPatterns(args []string) ([]glob.Glob, error) { + if len(args) == 0 { + args = []string{"**"} + } + pat := make([]glob.Glob,len(args)) + for i := range args { + if g, err := glob.Compile(args[i], '.', '/'); err != nil { + return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) + } else { + pat[i] = g + } + } + return pat, nil +} \ No newline at end of file diff --git a/main.go b/main.go index c67eaf7692e1d..4ac23b6061c6f 100644 --- a/main.go +++ b/main.go @@ -69,6 +69,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdKeys, cmd.CmdConvert, cmd.CmdDoctor, + cmd.Cmdembedded, } // Now adjust these commands to add our global configuration options diff --git a/modules/options/static.go b/modules/options/static.go index 1cf841ebb150e..be34b4e1300aa 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -112,3 +112,34 @@ func fileFromDir(name string) ([]byte, error) { return ioutil.ReadAll(f) } + +func Asset(name string) ([]byte, error) { + f, err := Assets.Open("/" + name) + if err != nil { + return nil, err + } + defer f.Close() + return ioutil.ReadAll(f) +} + +func AssetNames() []string { + realFS := Assets.(vfsgen۰FS) + var results = make([]string, 0, len(realFS)) + for k := range realFS { + results = append(results, k[1:]) + } + return results +} + +func AssetIsDir(name string) (bool, error) { + if f, err := Assets.Open("/" + name); err != nil { + return false, err + } else { + defer f.Close() + if fi, err := f.Stat(); err != nil { + return false, err + } else { + return fi.IsDir(), nil + } + } +} diff --git a/modules/public/static.go b/modules/public/static.go index 3e8865c3f93a2..880b29bab7da1 100644 --- a/modules/public/static.go +++ b/modules/public/static.go @@ -7,6 +7,8 @@ package public import ( + "io/ioutil" + "gitea.com/macaron/macaron" ) @@ -17,3 +19,34 @@ func Static(opts *Options) macaron.Handler { // used when in the options there is no FileSystem. return opts.staticHandler("") } + +func Asset(name string) ([]byte, error) { + f, err := Assets.Open("/" + name) + if err != nil { + return nil, err + } + defer f.Close() + return ioutil.ReadAll(f) +} + +func AssetNames() []string { + realFS := Assets.(vfsgen۰FS) + var results = make([]string, 0, len(realFS)) + for k := range realFS { + results = append(results, k[1:]) + } + return results +} + +func AssetIsDir(name string) (bool, error) { + if f, err := Assets.Open("/" + name); err != nil { + return false, err + } else { + defer f.Close() + if fi, err := f.Stat(); err != nil { + return false, err + } else { + return fi.IsDir(), nil + } + } +} diff --git a/modules/templates/static.go b/modules/templates/static.go index 435ccb1f95650..5bc4e33e1c936 100644 --- a/modules/templates/static.go +++ b/modules/templates/static.go @@ -229,3 +229,16 @@ func AssetNames() []string { } return results } + +func AssetIsDir(name string) (bool, error) { + if f, err := Assets.Open("/" + name); err != nil { + return false, err + } else { + defer f.Close() + if fi, err := f.Stat(); err != nil { + return false, err + } else { + return fi.IsDir(), nil + } + } +} From b9fc1d207c01c22cb8c3afdccf48a89fab397167 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 13:39:55 -0300 Subject: [PATCH 2/9] Implement extract command --- cmd/embedded.go | 163 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 126 insertions(+), 37 deletions(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index 41aab519d3df1..a2b48830a5140 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -7,9 +7,11 @@ package cmd import ( + "errors" "fmt" -// "os" -// "path/filepath" + "os" + "path/filepath" + "sort" "strings" "code.gitea.io/gitea/modules/log" @@ -33,13 +35,6 @@ var ( subcmdExtract, }, Flags: []cli.Flag{ - /* - cli.StringFlag{ - Name: "name, n", - Value: "**", - Usage: "glob pattern used to match files", - }, - */ cli.BoolFlag{ Name: "include-vendored,vendor", Usage: "Include files under public/vendor as well", @@ -62,36 +57,43 @@ var ( Name: "overwrite", Usage: "Overwrite files if they already exist", }, + cli.BoolFlag{ + Name: "rename", + Usage: "Rename files as {name}.bak if they already exist (overwrites previous .bak)", + }, cli.BoolFlag{ Name: "custom", - Usage: "Extract to the 'custom' directory", + Usage: "Extract to the 'custom' directory as per app.ini", }, cli.StringFlag{ - Name: "destination", - Usage: "Destination for the extracted files", + Name: "destination,dest-dir", + Usage: "Extract to the specified directory", }, }, } - sections map[string]*section - assets []asset + sections map[string]*section + assets []asset ) type section struct { - Path string - Names func() []string - IsDir func(string) (bool, error) + Path string + Names func() []string + IsDir func(string) (bool, error) + Asset func(string) ([]byte, error) } type asset struct { - Section *section - Name string + Section *section + Name string + Path string } func initEmbeddedExtractor(c *cli.Context) error { // Silence the console logger log.DelNamedLogger("console") + log.DelNamedLogger(log.DEFAULT) // Read configuration file setting.NewContext() @@ -100,34 +102,33 @@ func initEmbeddedExtractor(c *cli.Context) error { if err != nil { return err } - sections := make(map[string]*section,3) + sections := make(map[string]*section, 3) - sections["public"] = §ion{ Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir } - sections["options"] = §ion{ Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir } - sections["templates"] = §ion{ Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir } + sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} + sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} + sections["templates"] = §ion{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset} for _, sec := range sections { assets = append(assets, buildAssetList(sec, pats, c)...) } + // Sort assets + sort.SliceStable(assets, func(i, j int) bool { + return assets[i].Path < assets[j].Path + }) + return nil } -func runList(ctx *cli.Context) error { - if err := initEmbeddedExtractor(ctx); err != nil { +func runList(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { return err } - // fmt.Println("Using app.ini at", setting.CustomConf) - for _, asset := range assets { - fmt.Printf("- [%s] [%s]\n", asset.Section.Path, asset.Name) + for _, a := range assets { + fmt.Println(a.Path) } - fmt.Println("End of list.") - return nil -} -func runExtract(ctx *cli.Context) error { - fmt.Println("Not implemented") return nil } @@ -140,10 +141,12 @@ func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { !c.Bool("include-vendored") { continue } - matchName := "/"+sec.Path+"/"+name + matchName := "/" + sec.Path + "/" + name for _, g := range globs { if g.Match(matchName) { - results = append(results, asset{Section: sec, Name: name}) + results = append(results, asset{Section: sec, + Name: name, + Path: sec.Path + "/" + name}) break } } @@ -156,7 +159,7 @@ func getPatterns(args []string) ([]glob.Glob, error) { if len(args) == 0 { args = []string{"**"} } - pat := make([]glob.Glob,len(args)) + pat := make([]glob.Glob, len(args)) for i := range args { if g, err := glob.Compile(args[i], '.', '/'); err != nil { return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) @@ -165,4 +168,90 @@ func getPatterns(args []string) ([]glob.Glob, error) { } } return pat, nil -} \ No newline at end of file +} + +func runExtract(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { + return err + } + + destdir := "." + + if c.IsSet("destination") { + destdir = c.String("destination") + } else if c.Bool("custom") { + destdir = setting.CustomPath + fmt.Println("Using app.ini at", setting.CustomConf) + } + + if fi, err := os.Stat(destdir); err != nil { + return fmt.Errorf("%s: err", destdir) + } else if !fi.IsDir() { + return fmt.Errorf("%s does not exist or is not a directory.", destdir) + } + + fmt.Println("Extracting to", destdir) + + overwrite := c.Bool("overwrite") + rename := c.Bool("rename") + + for _, a := range assets { + if err := extractAsset(destdir, a, overwrite, rename); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) + } + } + + return nil +} + +func extractAsset(d string, a asset, overwrite, rename bool) error { + dest := d + "/" + a.Path + dir := filepath.Dir(dest) + + data, err := a.Section.Asset(a.Name) + if err != nil { + return fmt.Errorf("%s: %v", a.Path, err) + } + + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("%s: %v", dir, err) + } + + perms := os.ModePerm & 0666 + + fi, err := os.Lstat(dest) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s: %v", dest, err) + } + } else if !overwrite && !rename { + fmt.Fprintf(os.Stderr, "%s already exists; skipped.\n", dest) + return nil + } else if !fi.Mode().IsRegular() { + return fmt.Errorf("%s already exists and is not a regular file", dest) + } else if rename { + if err := os.Rename(dest, dest+".bak"); err != nil { + return fmt.Errorf("Error creating backup for %s: %v", dest, err) + } + // Attempt to respect file permissions mask (even if user:group will be set anew) + perms = fi.Mode() + } + + file, err := os.OpenFile(dest, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, perms) + if err != nil { + return fmt.Errorf("%s: %v", dest, err) + } + defer file.Close() + + for len(data) != 0 { + written, err := file.Write(data) + if err != nil { + return fmt.Errorf("%s: %v", dest, err) + } + data = data[written:] + } + + fmt.Println(dest) + + return nil +} From cad82a6243e3c1e8e42158be5be069ad9afd31f3 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 14:03:03 -0300 Subject: [PATCH 3/9] Fix nits and force args on extract --- cmd/embedded.go | 104 +++++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 41 deletions(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index a2b48830a5140..b0db79a32948d 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -121,60 +121,42 @@ func initEmbeddedExtractor(c *cli.Context) error { } func runList(c *cli.Context) error { - if err := initEmbeddedExtractor(c); err != nil { + if err := runListDo(c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) return err } - - for _, a := range assets { - fmt.Println(a.Path) - } - return nil } -func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { - var results = make([]asset, 0, 64) - for _, name := range sec.Names() { - if isdir, err := sec.IsDir(name); !isdir && err == nil { - if sec.Path == "public" && - strings.HasPrefix(name, "vendor/") && - !c.Bool("include-vendored") { - continue - } - matchName := "/" + sec.Path + "/" + name - for _, g := range globs { - if g.Match(matchName) { - results = append(results, asset{Section: sec, - Name: name, - Path: sec.Path + "/" + name}) - break - } - } - } +func runExtract(c *cli.Context) error { + if err := runExtractDo(c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err } - return results + return nil } -func getPatterns(args []string) ([]glob.Glob, error) { - if len(args) == 0 { - args = []string{"**"} +func runListDo(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { + return err } - pat := make([]glob.Glob, len(args)) - for i := range args { - if g, err := glob.Compile(args[i], '.', '/'); err != nil { - return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) - } else { - pat[i] = g - } + + for _, a := range assets { + fmt.Println(a.Path) } - return pat, nil + + return nil } -func runExtract(c *cli.Context) error { +func runExtractDo(c *cli.Context) error { if err := initEmbeddedExtractor(c); err != nil { return err } + if len(c.Args()) == 0 { + return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") + } + destdir := "." if c.IsSet("destination") { @@ -185,18 +167,19 @@ func runExtract(c *cli.Context) error { } if fi, err := os.Stat(destdir); err != nil { - return fmt.Errorf("%s: err", destdir) + return fmt.Errorf("%s: %s", destdir, err) } else if !fi.IsDir() { return fmt.Errorf("%s does not exist or is not a directory.", destdir) } - fmt.Println("Extracting to", destdir) + fmt.Printf("Extracting to %s:\n", destdir) overwrite := c.Bool("overwrite") rename := c.Bool("rename") for _, a := range assets { if err := extractAsset(destdir, a, overwrite, rename); err != nil { + // Non-fatal error fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) } } @@ -225,7 +208,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return fmt.Errorf("%s: %v", dest, err) } } else if !overwrite && !rename { - fmt.Fprintf(os.Stderr, "%s already exists; skipped.\n", dest) + fmt.Printf("%s already exists; skipped.\n", dest) return nil } else if !fi.Mode().IsRegular() { return fmt.Errorf("%s already exists and is not a regular file", dest) @@ -255,3 +238,42 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { return nil } + +func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { + var results = make([]asset, 0, 64) + for _, name := range sec.Names() { + if isdir, err := sec.IsDir(name); !isdir && err == nil { + if sec.Path == "public" && + strings.HasPrefix(name, "vendor/") && + !c.Bool("include-vendored") { + continue + } + matchName := "/" + sec.Path + "/" + name + for _, g := range globs { + if g.Match(matchName) { + results = append(results, asset{Section: sec, + Name: name, + Path: sec.Path + "/" + name}) + break + } + } + } + } + return results +} + +func getPatterns(args []string) ([]glob.Glob, error) { + if len(args) == 0 { + args = []string{"**"} + } + pat := make([]glob.Glob, len(args)) + for i := range args { + if g, err := glob.Compile(args[i], '.', '/'); err != nil { + return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) + } else { + pat[i] = g + } + } + return pat, nil +} + From f472e768f298307d3d203fa56285b4ecd975baae Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 20:13:25 -0300 Subject: [PATCH 4/9] Add !bindata stub, support Windows, fmt --- cmd/embedded.go | 31 ++++++++++++++++++------------- cmd/embedded_stub.go | 30 ++++++++++++++++++++++++++++++ modules/public/static.go | 2 +- 3 files changed, 49 insertions(+), 14 deletions(-) create mode 100644 cmd/embedded_stub.go diff --git a/cmd/embedded.go b/cmd/embedded.go index b0db79a32948d..66c83a2f0f981 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -29,7 +29,7 @@ var ( Cmdembedded = cli.Command{ Name: "embedded", Usage: "Extract embedded resources", - Description: "A command for extracting embedded resources, like templates and images.", + Description: "A command for extracting embedded resources, like templates and images", Subcommands: []cli.Command{ subcmdList, subcmdExtract, @@ -166,10 +166,20 @@ func runExtractDo(c *cli.Context) error { fmt.Println("Using app.ini at", setting.CustomConf) } - if fi, err := os.Stat(destdir); err != nil { + fi, err := os.Stat(destdir) + if errors.Is(err, os.ErrNotExist) { + // In case Windows users attempt to provide a forward-slash path + wdestdir := filepath.FromSlash(destdir) + if wfi, werr := os.Stat(wdestdir); werr == nil { + destdir = wdestdir + fi = wfi + err = nil + } + } + if err != nil { return fmt.Errorf("%s: %s", destdir, err) } else if !fi.IsDir() { - return fmt.Errorf("%s does not exist or is not a directory.", destdir) + return fmt.Errorf("%s is not a directory.", destdir) } fmt.Printf("Extracting to %s:\n", destdir) @@ -188,7 +198,7 @@ func runExtractDo(c *cli.Context) error { } func extractAsset(d string, a asset, overwrite, rename bool) error { - dest := d + "/" + a.Path + dest := filepath.Join(d, filepath.FromSlash(a.Path)) dir := filepath.Dir(dest) data, err := a.Section.Asset(a.Name) @@ -211,7 +221,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { fmt.Printf("%s already exists; skipped.\n", dest) return nil } else if !fi.Mode().IsRegular() { - return fmt.Errorf("%s already exists and is not a regular file", dest) + return fmt.Errorf("%s already exists, but it's not a regular file", dest) } else if rename { if err := os.Rename(dest, dest+".bak"); err != nil { return fmt.Errorf("Error creating backup for %s: %v", dest, err) @@ -226,12 +236,8 @@ func extractAsset(d string, a asset, overwrite, rename bool) error { } defer file.Close() - for len(data) != 0 { - written, err := file.Write(data) - if err != nil { - return fmt.Errorf("%s: %v", dest, err) - } - data = data[written:] + if _, err = file.Write(data); err != nil { + return fmt.Errorf("%s: %v", dest, err) } fmt.Println(dest) @@ -268,7 +274,7 @@ func getPatterns(args []string) ([]glob.Glob, error) { } pat := make([]glob.Glob, len(args)) for i := range args { - if g, err := glob.Compile(args[i], '.', '/'); err != nil { + if g, err := glob.Compile(args[i], '/'); err != nil { return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) } else { pat[i] = g @@ -276,4 +282,3 @@ func getPatterns(args []string) ([]glob.Glob, error) { } return pat, nil } - diff --git a/cmd/embedded_stub.go b/cmd/embedded_stub.go new file mode 100644 index 0000000000000..1f9af7b86b123 --- /dev/null +++ b/cmd/embedded_stub.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !bindata + +package cmd + +import ( + "fmt" + "os" + + "github.com/urfave/cli" +) + +// Cmdembedded represents the available extract sub-command. +var ( + Cmdembedded = cli.Command{ + Name: "embedded", + Usage: "Extract embedded resources", + Description: "A command for extracting embedded resources, like templates and images", + Action: extractorNotImplemented, + } +) + +func extractorNotImplemented(c *cli.Context) error { + err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") + fmt.Fprintf(os.Stderr, "%s\n", err) + return err +} diff --git a/modules/public/static.go b/modules/public/static.go index 880b29bab7da1..76050632c9f11 100644 --- a/modules/public/static.go +++ b/modules/public/static.go @@ -8,7 +8,7 @@ package public import ( "io/ioutil" - + "gitea.com/macaron/macaron" ) From db111c48305539270197a9fb86164765a33ea3e9 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 21:08:31 -0300 Subject: [PATCH 5/9] fix vendored flag --- cmd/embedded.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index 66c83a2f0f981..01cec4197a399 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -34,18 +34,18 @@ var ( subcmdList, subcmdExtract, }, - Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "include-vendored,vendor", - Usage: "Include files under public/vendor as well", - }, - }, } subcmdList = cli.Command{ Name: "list", Usage: "List files matching the given pattern", Action: runList, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, + }, } subcmdExtract = cli.Command{ @@ -53,6 +53,10 @@ var ( Usage: "Extract resources", Action: runExtract, Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, cli.BoolFlag{ Name: "overwrite", Usage: "Overwrite files if they already exist", From 5b69be9e7cd2d975756faa5bbd28d154c50dd7c8 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 21:18:03 -0300 Subject: [PATCH 6/9] Remove leading slash for matching --- cmd/embedded.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/embedded.go b/cmd/embedded.go index 01cec4197a399..6715db3b76b40 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -258,7 +258,7 @@ func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { !c.Bool("include-vendored") { continue } - matchName := "/" + sec.Path + "/" + name + matchName := sec.Path + "/" + name for _, g := range globs { if g.Match(matchName) { results = append(results, asset{Section: sec, From 91010b045a1bd198ba6825cbe3348bf0e9453c34 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 22:06:47 -0300 Subject: [PATCH 7/9] Add docs --- .../doc/advanced/cmd-embedded.en-us.md | 115 ++++++++++++++++++ .../doc/advanced/customizing-gitea.en-us.md | 19 ++- 2 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 docs/content/doc/advanced/cmd-embedded.en-us.md diff --git a/docs/content/doc/advanced/cmd-embedded.en-us.md b/docs/content/doc/advanced/cmd-embedded.en-us.md new file mode 100644 index 0000000000000..90c4a3f6b6430 --- /dev/null +++ b/docs/content/doc/advanced/cmd-embedded.en-us.md @@ -0,0 +1,115 @@ +--- +date: "2020-01-25T21:00:00-03:00" +title: "Embedded data extraction tool" +slug: "cmd-embedded" +weight: 40 +toc: true +draft: false +menu: + sidebar: + parent: "advanced" + name: "Embedded data extraction tool" + weight: 40 + identifier: "cmd-embedded" +--- + +# Embedded data extraction tool + +Gitea's executable contains all the resources required to run: templates, images, style-sheets +and translations. Any of them can be overridden by placing a replacement in a matching path +inside the `custom` directory (see [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}})). + +To obtain a copy of the embedded resources ready for editing, the `embedded` command from the CLI +can be used from the OS shell interface. + +## List resources + +To list resources embedded in Gitea's executable, use the following syntax: + +``` +gitea embedded list [--include-vendored] [patterns...] +``` + +The `--include-vendored` flag makes the command include vendored files, which are +normally excluded; that is, files from external libraries that are required for Gitea +(e.g. [font-awesome](https://fontawesome.com/), [octicons](https://octicons.github.com/), etc). + +A list of file search patterns can be provided. Gitea uses [gobwas/glob](https://github.com/gobwas/glob) +for its glob syntax. Here are some examples: + +- List all template files, in any virtual directory: `**.tmpl` +- List all mail template files: `templates/mail/**.tmpl` +- List all files inside `public/img`: `public/img/**` + +Don't forget to use quotes for the patterns, as spaces, `*` and other characters might have +a special significance for your command shell. + +If no pattern is provided, all files are listed. + +#### Example + +Listing all embedded files with `openid` in their path: + +``` +$ gitea embedded list '**openid**' +public/img/auth/openid_connect.png +public/img/openid-16x16.png +templates/user/auth/finalize_openid.tmpl +templates/user/auth/signin_openid.tmpl +templates/user/auth/signup_openid_connect.tmpl +templates/user/auth/signup_openid_navbar.tmpl +templates/user/auth/signup_openid_register.tmpl +templates/user/settings/security_openid.tmpl +``` + +## Extract resources + +To extract resources embedded in Gitea's executable, use the following syntax: + +``` +gitea [--config {file}] embedded extract [--destination {dir}|--custom] [--overwrite|--rename] [--include-vendored] {patterns...} +``` + +The `--config` option tells gitea the location of the `app.ini` configuration file if +it's not in its default location. This option is only used with the `--custom` flag. + +The `--destination` option tells gitea the directory where the files must be extracted to. +The default is the current directory. + +The `--custom` flag tells gitea to extract the files directly into the `custom` directory +For this to work, the command needs to know the location of the `app.ini` configuration +file (`--config`) and, depending of the configuration, be ran from the directory where +gitea normally starts. See [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) for details. + +The `--overwrite` flag allows any existing files in the destination directory to be overwritten. + +The `--rename` flag tells gitea to rename any existing files in the destination directory +as `filename.bak`. Previous `.bak` files are overwritten. + +At least one file search pattern must be provided; see `list` subcomand above for pattern +syntax and examples. + +#### Important notice + +Make sure to **only extract those files that require customization**. Files that +are present in the `custom` directory are not upgraded by Gitea's upgrade process. +When Gitea is upgraded to a new version (by replacing the executable), many of the +embedded files will suffer changes. Gitea will honor and use any files found +in the `custom` directory, even if they are old and incompatible. + +#### Example + +Extracting mail templates to a temporary directory: + +``` +$ mkdir tempdir +$ gitea embedded extract --destination tempdir 'templates/mail/**.tmpl' +Extracting to tempdir: +tempdir/templates/mail/auth/activate.tmpl +tempdir/templates/mail/auth/activate_email.tmpl +tempdir/templates/mail/auth/register_notify.tmpl +tempdir/templates/mail/auth/reset_passwd.tmpl +tempdir/templates/mail/issue/assigned.tmpl +tempdir/templates/mail/issue/default.tmpl +tempdir/templates/mail/notify/collaborator.tmpl +``` diff --git a/docs/content/doc/advanced/customizing-gitea.en-us.md b/docs/content/doc/advanced/customizing-gitea.en-us.md index f42ade7799493..73127dd81a3cd 100644 --- a/docs/content/doc/advanced/customizing-gitea.en-us.md +++ b/docs/content/doc/advanced/customizing-gitea.en-us.md @@ -57,14 +57,21 @@ the url `http://gitea.domain.tld/image.png`. Place the png image at the following path: `custom/public/img/avatar_default.png` -## Customizing Gitea pages +## Customizing Gitea pages and resources -The `custom/templates` folder allows changing every single page of Gitea. Templates -to override can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from `master` branch. Make sure to copy templates from same release you are using). Override by -making a copy of the file under `custom/templates` using a full path structure -matching source. +Gitea's executable contains all the resources required to run: templates, images, style-sheets +and translations. Any of them can be overridden by placing a replacement in a matching path +inside the `custom` directory. For example, to replace the default `.gitignore` provided +for C++ repositories, we want to replace `options/gitignore/C++`. To do this, a replacement +must be placed in `custom/options/gitignore/C++` (see about the location of the `custom` +directory at the top of this document). -Any statement contained inside `{{` and `}}` are Gitea's template syntax and +Every single page of Gitea can be changed. Dynamic content is generated using [go templates](https://golang.org/pkg/html/template/), +which can be modified by placing replacements below the `custom/templates` directory. + +To obtain any embedded file (including templates), the [`gitea embedded` tool]({{< relref "doc/advanced/cmd-embedded.en-us.md" >}}) can be used. Alternatively, they can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from the `master` branch. Make sure to use templates compatible with the release you are using). + +Be aware that any statement contained inside `{{` and `}}` are Gitea's template syntax and shouldn't be touched without fully understanding these components. ### Customizing startpage / homepage From 7d8ee85a648062218729c81e0899a970f350b415 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 25 Jan 2020 22:43:07 -0300 Subject: [PATCH 8/9] Fix typos --- docs/content/doc/advanced/cmd-embedded.en-us.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/content/doc/advanced/cmd-embedded.en-us.md b/docs/content/doc/advanced/cmd-embedded.en-us.md index 90c4a3f6b6430..e036d17498152 100644 --- a/docs/content/doc/advanced/cmd-embedded.en-us.md +++ b/docs/content/doc/advanced/cmd-embedded.en-us.md @@ -22,7 +22,7 @@ inside the `custom` directory (see [Customizing Gitea]({{< relref "doc/advanced/ To obtain a copy of the embedded resources ready for editing, the `embedded` command from the CLI can be used from the OS shell interface. -## List resources +## Listing resources To list resources embedded in Gitea's executable, use the following syntax: @@ -42,7 +42,7 @@ for its glob syntax. Here are some examples: - List all files inside `public/img`: `public/img/**` Don't forget to use quotes for the patterns, as spaces, `*` and other characters might have -a special significance for your command shell. +a special meaning for your command shell. If no pattern is provided, all files are listed. @@ -62,7 +62,7 @@ templates/user/auth/signup_openid_register.tmpl templates/user/settings/security_openid.tmpl ``` -## Extract resources +## Extracting resources To extract resources embedded in Gitea's executable, use the following syntax: @@ -76,7 +76,7 @@ it's not in its default location. This option is only used with the `--custom` f The `--destination` option tells gitea the directory where the files must be extracted to. The default is the current directory. -The `--custom` flag tells gitea to extract the files directly into the `custom` directory +The `--custom` flag tells gitea to extract the files directly into the `custom` directory. For this to work, the command needs to know the location of the `app.ini` configuration file (`--config`) and, depending of the configuration, be ran from the directory where gitea normally starts. See [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) for details. From 79c18cac486dc26ea410b53fc6b7f1fc4f29c7d1 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Sat, 1 Feb 2020 12:28:23 -0300 Subject: [PATCH 9/9] Add embedded view command --- cmd/embedded.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cmd/embedded.go b/cmd/embedded.go index 6715db3b76b40..363b85c066eaa 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -32,6 +32,7 @@ var ( Description: "A command for extracting embedded resources, like templates and images", Subcommands: []cli.Command{ subcmdList, + subcmdView, subcmdExtract, }, } @@ -48,6 +49,18 @@ var ( }, } + subcmdView = cli.Command{ + Name: "view", + Usage: "View a file matching the given pattern", + Action: runView, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "include-vendored,vendor", + Usage: "Include files under public/vendor as well", + }, + }, + } + subcmdExtract = cli.Command{ Name: "extract", Usage: "Extract resources", @@ -132,6 +145,14 @@ func runList(c *cli.Context) error { return nil } +func runView(c *cli.Context) error { + if err := runViewDo(c); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + return err + } + return nil +} + func runExtract(c *cli.Context) error { if err := runExtractDo(c); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) @@ -152,6 +173,29 @@ func runListDo(c *cli.Context) error { return nil } +func runViewDo(c *cli.Context) error { + if err := initEmbeddedExtractor(c); err != nil { + return err + } + + if len(assets) == 0 { + return fmt.Errorf("No files matched the given pattern") + } else if len(assets) > 1 { + return fmt.Errorf("Too many files matched the given pattern; try to be more specific") + } + + data, err := assets[0].Section.Asset(assets[0].Name) + if err != nil { + return fmt.Errorf("%s: %v", assets[0].Path, err) + } + + if _, err = os.Stdout.Write(data); err != nil { + return fmt.Errorf("%s: %v", assets[0].Path, err) + } + + return nil +} + func runExtractDo(c *cli.Context) error { if err := initEmbeddedExtractor(c); err != nil { return err