From 94426b6723a8d3e387dbc8ea5e450b551b2aef9b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 24 Jun 2019 20:22:49 +0100 Subject: [PATCH 01/11] Add cmd to allow generalised mapping of environment variables to Gitea config --- cmd/environment_to_ini.go | 92 ++++++++++++++++++++++++++++++++++ docker/root/etc/s6/gitea/setup | 2 + main.go | 1 + 3 files changed, 95 insertions(+) create mode 100644 cmd/environment_to_ini.go diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go new file mode 100644 index 000000000000..531d1b1d6e4c --- /dev/null +++ b/cmd/environment_to_ini.go @@ -0,0 +1,92 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "os" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "github.com/Unknwon/com" + "github.com/urfave/cli" + ini "gopkg.in/ini.v1" +) + +// EnvironmentPrefix environment variables prefixed with this represent ini values to write +const EnvironmentPrefix = "GITEA:" + +// Separator is the character that will separate section from key name +const Separator = ":" + +// CmdEnvironmentToIni represents the command to use a provided environment to update the configuration ini +var CmdEnvironmentToIni = cli.Command{ + Name: "enviroment-to-ini", + Usage: "Use provided environment to update configuration ini", + Action: runEnvironmentToIni, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "out, o", + Value: "", + Usage: "Destination file to write to", + }, + }, +} + +func runEnvironmentToIni(c *cli.Context) error { + cfg := ini.Empty() + if com.IsFile(setting.CustomConf) { + if err := cfg.Append(setting.CustomConf); err != nil { + log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err) + } + } else { + log.Warn("Custom config '%s' not found, ignore this if you're running first time", setting.CustomConf) + } + cfg.NameMapper = ini.AllCapsUnderscore + + for _, kv := range os.Environ() { + idx := strings.IndexByte(kv, '=') + if idx < 0 { + continue + } + eKey := kv[:idx] + value := kv[idx+1:] + if !strings.HasPrefix(eKey, EnvironmentPrefix) { + continue + } + parts := strings.Split(eKey, Separator) + if len(parts) != 3 { + continue + } + sectionName := parts[1] + keyName := parts[2] + if len(sectionName) == 0 || len(keyName) == 0 { + continue + } + section, err := cfg.GetSection(sectionName) + if err != nil { + section, err = cfg.NewSection(sectionName) + if err != nil { + log.Error("Error creating section: %s : %v", sectionName, err) + continue + } + } + key := section.Key(keyName) + if key == nil { + key, err = section.NewKey(keyName, value) + if err != nil { + log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, value, err) + continue + } + } + key.SetValue(value) + } + destination := c.String("out") + if len(destination) == 0 { + destination = setting.CustomConf + } + err := cfg.SaveTo(destination) + return err +} diff --git a/docker/root/etc/s6/gitea/setup b/docker/root/etc/s6/gitea/setup index c4fbf5d65ea1..6dbae0a7f98c 100755 --- a/docker/root/etc/s6/gitea/setup +++ b/docker/root/etc/s6/gitea/setup @@ -43,6 +43,8 @@ if [ ! -f ${GITEA_CUSTOM}/conf/app.ini ]; then SECRET_KEY=${SECRET_KEY:-""} \ envsubst < /etc/templates/app.ini > ${GITEA_CUSTOM}/conf/app.ini + gitea environment-to-ini -c ${GITEA_CUSTOM}/conf/app.ini + chown ${USER}:git ${GITEA_CUSTOM}/conf/app.ini fi diff --git a/main.go b/main.go index 30dbf2766224..1220a8e6c14e 100644 --- a/main.go +++ b/main.go @@ -68,6 +68,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdMigrate, cmd.CmdKeys, cmd.CmdConvert, + cmd.CmdEnvironmentToIni, } // Now adjust these commands to add our global configuration options From 890d539b2b093534f45ae5cd35b24a4a4e0cc946 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 24 Jun 2019 21:01:44 +0100 Subject: [PATCH 02/11] spelling mistake --- cmd/environment_to_ini.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go index 531d1b1d6e4c..72603ddb18b9 100644 --- a/cmd/environment_to_ini.go +++ b/cmd/environment_to_ini.go @@ -23,7 +23,7 @@ const Separator = ":" // CmdEnvironmentToIni represents the command to use a provided environment to update the configuration ini var CmdEnvironmentToIni = cli.Command{ - Name: "enviroment-to-ini", + Name: "environment-to-ini", Usage: "Use provided environment to update configuration ini", Action: runEnvironmentToIni, Flags: []cli.Flag{ From 51338a1d0cfae0dcf14560ddc9c2da42777bdfe8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 24 Jun 2019 21:14:55 +0100 Subject: [PATCH 03/11] Use full path to gitea --- docker/root/etc/s6/gitea/setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/root/etc/s6/gitea/setup b/docker/root/etc/s6/gitea/setup index 6dbae0a7f98c..a10791bddee6 100755 --- a/docker/root/etc/s6/gitea/setup +++ b/docker/root/etc/s6/gitea/setup @@ -43,7 +43,7 @@ if [ ! -f ${GITEA_CUSTOM}/conf/app.ini ]; then SECRET_KEY=${SECRET_KEY:-""} \ envsubst < /etc/templates/app.ini > ${GITEA_CUSTOM}/conf/app.ini - gitea environment-to-ini -c ${GITEA_CUSTOM}/conf/app.ini + /app/gitea/gitea environment-to-ini -c ${GITEA_CUSTOM}/conf/app.ini chown ${USER}:git ${GITEA_CUSTOM}/conf/app.ini fi From 64c15c2999ae628f350386e3906eab4b95627e9f Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 24 Jun 2019 22:07:23 +0100 Subject: [PATCH 04/11] Move to use double underscore as a separator --- cmd/environment_to_ini.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go index 72603ddb18b9..3cadde1b5b36 100644 --- a/cmd/environment_to_ini.go +++ b/cmd/environment_to_ini.go @@ -16,10 +16,10 @@ import ( ) // EnvironmentPrefix environment variables prefixed with this represent ini values to write -const EnvironmentPrefix = "GITEA:" +const EnvironmentPrefix = "GITEA__" // Separator is the character that will separate section from key name -const Separator = ":" +const Separator = "__" // CmdEnvironmentToIni represents the command to use a provided environment to update the configuration ini var CmdEnvironmentToIni = cli.Command{ From f73377b1e0c1902e1bfb38f9a257f8305960499f Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 26 Jun 2019 20:38:12 +0100 Subject: [PATCH 05/11] Add encoding to the environment strings --- cmd/environment_to_ini.go | 70 ++++++++++++++++++++++++++++++---- cmd/environment_to_ini_test.go | 67 ++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 cmd/environment_to_ini_test.go diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go index 3cadde1b5b36..4d04c6e6dd2d 100644 --- a/cmd/environment_to_ini.go +++ b/cmd/environment_to_ini.go @@ -6,6 +6,8 @@ package cmd import ( "os" + "regexp" + "strconv" "strings" "code.gitea.io/gitea/modules/log" @@ -56,13 +58,8 @@ func runEnvironmentToIni(c *cli.Context) error { if !strings.HasPrefix(eKey, EnvironmentPrefix) { continue } - parts := strings.Split(eKey, Separator) - if len(parts) != 3 { - continue - } - sectionName := parts[1] - keyName := parts[2] - if len(sectionName) == 0 || len(keyName) == 0 { + sectionName, keyName := DecodeSectionKey(eKey) + if len(keyName) == 0 { continue } section, err := cfg.GetSection(sectionName) @@ -90,3 +87,62 @@ func runEnvironmentToIni(c *cli.Context) error { err := cfg.SaveTo(destination) return err } + +const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_" + +var escapeRegex = regexp.MustCompile(escapeRegexpString) + +// DecodeSectionKey will decode a portable string encoded Section__Key pair +// Portable strings are considered to be of the form [A-Z0-9_]* +// We will encode a disallowed value as the UTF8 byte string preceded by _0X and +// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.' +// Section and Key are separated by a plain '__'. +// The entire section can be encoded as a UTF8 byte string +func DecodeSectionKey(encoded string) (string, string) { + section := "" + key := "" + + inKey := false + last := 0 + escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1) + for _, unescapeIdx := range escapeStringIndices { + preceding := encoded[last:unescapeIdx[0]] + if !inKey { + if splitter := strings.Index(preceding, "__"); splitter > -1 { + section += preceding[:splitter] + inKey = true + key += preceding[splitter+2:] + } else { + section += preceding + } + } else { + key += preceding + } + toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1] + decodedBytes := make([]byte, len(toDecode)/2) + for i := 0; i < len(toDecode)/2; i++ { + // Can ignore error here as we know these should be hexadecimal from the regexp + byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0) + decodedBytes[i] = byte(byteInt) + } + if inKey { + key += string(decodedBytes) + } else { + section += string(decodedBytes) + } + last = unescapeIdx[1] + } + remaining := encoded[last:] + if !inKey { + if splitter := strings.Index(remaining, "__"); splitter > -1 { + section += remaining[:splitter] + inKey = true + key += remaining[splitter+2:] + } else { + section += remaining + } + } else { + key += remaining + } + return section, key +} diff --git a/cmd/environment_to_ini_test.go b/cmd/environment_to_ini_test.go new file mode 100644 index 000000000000..041c4ba57d6e --- /dev/null +++ b/cmd/environment_to_ini_test.go @@ -0,0 +1,67 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import "testing" + +func TestDecodeSectionKey(t *testing.T) { + type args struct { + encoded string + } + tests := []struct { + name string + encoded string + section string + key string + }{ + { + name: "Simple", + encoded: "Section__Key", + section: "Section", + key: "Key", + }, + { + name: "LessSimple", + encoded: "Section_SubSection__Key_SubKey", + section: "Section_SubSection", + key: "Key_SubKey", + }, + { + name: "OneDotOneDash", + encoded: "Section_0X2E_SubSection__Key_0X2D_SubKey", + section: "Section.SubSection", + key: "Key-SubKey", + }, + { + name: "OneDotOneEncodedOneDash", + encoded: "Section_0X2E_0X2E_Sub_0X2D_Section__Key_0X2D_SubKey", + section: "Section.0X2E_Sub-Section", + key: "Key-SubKey", + }, + { + name: "EncodedUnderscore", + encoded: "Section__0X5F_0X2E_Sub_0X2D_Section__Key_0X2D__0X2D_SubKey", + section: "Section__0X2E_Sub-Section", + key: "Key--SubKey", + }, + { + name: "EncodedUtf8", + encoded: "Section__0XE280A6_Sub_0X2D_Section__Key_0X2D__0X2D_SubKey", + section: "Section_…Sub-Section", + key: "Key--SubKey", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotSection, gotKey := DecodeSectionKey(tt.encoded) + if gotSection != tt.section { + t.Errorf("DecodeSectionKey() gotSection = %v, want %v", gotSection, tt.section) + } + if gotKey != tt.key { + t.Errorf("DecodeSectionKey() gotKey = %v, want %v", gotKey, tt.key) + } + }) + } +} From 730e3004e407173b155264e0dfdf9f75633a9f1b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 26 Jun 2019 20:53:36 +0100 Subject: [PATCH 06/11] Remove unused separator --- cmd/environment_to_ini.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go index 4d04c6e6dd2d..158d30c166bb 100644 --- a/cmd/environment_to_ini.go +++ b/cmd/environment_to_ini.go @@ -20,9 +20,6 @@ import ( // EnvironmentPrefix environment variables prefixed with this represent ini values to write const EnvironmentPrefix = "GITEA__" -// Separator is the character that will separate section from key name -const Separator = "__" - // CmdEnvironmentToIni represents the command to use a provided environment to update the configuration ini var CmdEnvironmentToIni = cli.Command{ Name: "environment-to-ini", From e0ca8362a22b325acd4c34b7fa5b7be9541f758b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 26 Jun 2019 20:56:25 +0100 Subject: [PATCH 07/11] Add Documentation --- .../doc/installation/with-docker.en-us.md | 4 ++++ docs/content/doc/usage/command-line.en-us.md | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/content/doc/installation/with-docker.en-us.md b/docs/content/doc/installation/with-docker.en-us.md index a403d1877830..26a60f2d52fc 100644 --- a/docs/content/doc/installation/with-docker.en-us.md +++ b/docs/content/doc/installation/with-docker.en-us.md @@ -261,6 +261,10 @@ You can configure some of Gitea's settings via environment variables: * `USER_UID`: **1000**: The UID (Unix user ID) of the user that runs Gitea within the container. Match this to the UID of the owner of the `/data` volume if using host volumes (this is not necessary with named volumes). * `USER_GID`: **1000**: The GID (Unix group ID) of the user that runs Gitea within the container. Match this to the GID of the owner of the `/data` volume if using host volumes (this is not necessary with named volumes). +Any other configuration value can be set using environment variables +of the form: `GITEA__SECTION_NAME__KEY_NAME`. See the +`environment-to-ini` command for more information. + # Customization Customization files described [here](https://docs.gitea.io/en-us/customizing-gitea/) should diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 0955680af220..7b676f8ac0a7 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -281,3 +281,22 @@ provided key. You should also set the value NB: opensshd requires the gitea program to be owned by root and not writable by group or others. The program must be specified by an absolute path. + +#### environment-to-ini + +As a helper to allow docker users to update the gitea configuration +through the environment, this command allows environment variables to +be mapped to values in the ini. + +Environment variables of the form `GITEA__SECTION_NAME__KEY_NAME` +will be mapped to the ini section `[section_name]` and the key +`KEY_NAME` with the value as provided. + +Environment variables are usually restricted to a reduced character +set `0-9A-Z_` - in order to allow the setting of sections with +characters outside of that set, they should be escaped as following: +`_0X2E_` for `.`. The entire section and key names can be escaped as +a UTF8 byte string if necessary. + +- Options: + - `--out name`, `-o name`: Name of the adjusted ini file to be created. Optional. (default: The gitea conf file will be changed in place). From f7d10eaff34bf677e2394c7699f22ba2f840f360 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 26 Jun 2019 21:03:28 +0100 Subject: [PATCH 08/11] Remove unused args struct --- cmd/environment_to_ini_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/environment_to_ini_test.go b/cmd/environment_to_ini_test.go index 041c4ba57d6e..1baf411990f0 100644 --- a/cmd/environment_to_ini_test.go +++ b/cmd/environment_to_ini_test.go @@ -7,9 +7,6 @@ package cmd import "testing" func TestDecodeSectionKey(t *testing.T) { - type args struct { - encoded string - } tests := []struct { name string encoded string From 3f5e1ba35e9a9eea0da7ea7a89990fcd63ce50f3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 17 Aug 2019 13:43:33 +0100 Subject: [PATCH 09/11] Provide example as per @silverwind --- docs/content/doc/usage/command-line.en-us.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index 7b676f8ac0a7..8550536051c7 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -296,7 +296,22 @@ Environment variables are usually restricted to a reduced character set `0-9A-Z_` - in order to allow the setting of sections with characters outside of that set, they should be escaped as following: `_0X2E_` for `.`. The entire section and key names can be escaped as -a UTF8 byte string if necessary. +a UTF8 byte string if necessary. E.g. to configure: + +```ini +... + +[log.console] +COLORIZE=false +STDERR=true + +... +``` + +You would set the environment variables: `GITEA__LOG_0x2E_CONSOLE__COLORIZE=false` +and `GITEA__LOG_0x2E_CONSOLE__STDERR=false`. Other examples can be found +on the configuration cheat sheet. + - Options: - `--out name`, `-o name`: Name of the adjusted ini file to be created. Optional. (default: The gitea conf file will be changed in place). From 019e1247eca219eb73bdfe256bf575ab4c133477 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 9 Oct 2019 18:33:15 +0100 Subject: [PATCH 10/11] Fix Unknwon to unknwon in cmd/environment_to_ini --- cmd/environment_to_ini.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go index 158d30c166bb..36f4b2b6eae9 100644 --- a/cmd/environment_to_ini.go +++ b/cmd/environment_to_ini.go @@ -12,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "github.com/Unknwon/com" + "github.com/unknwon/com" "github.com/urfave/cli" ini "gopkg.in/ini.v1" ) From f1570621c9167dd3f90c824ce096edba0d7285ff Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 9 Oct 2019 18:40:12 +0100 Subject: [PATCH 11/11] Remove Prefix before passing to DecodeSectionKey --- cmd/environment_to_ini.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/environment_to_ini.go b/cmd/environment_to_ini.go index 36f4b2b6eae9..baf9118f4e29 100644 --- a/cmd/environment_to_ini.go +++ b/cmd/environment_to_ini.go @@ -55,6 +55,7 @@ func runEnvironmentToIni(c *cli.Context) error { if !strings.HasPrefix(eKey, EnvironmentPrefix) { continue } + eKey = eKey[len(EnvironmentPrefix):] sectionName, keyName := DecodeSectionKey(eKey) if len(keyName) == 0 { continue