Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make environment-to-ini support loading key value from file #24832

Merged
merged 5 commits into from
May 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 15 additions & 105 deletions contrib/environment-to-ini/environment-to-ini.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,14 @@ package main

import (
"os"
"regexp"
"strconv"
"strings"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"

"github.com/urfave/cli"
ini "gopkg.in/ini.v1"
"gopkg.in/ini.v1"
)

// EnvironmentPrefix environment variables prefixed with this represent ini values to write
Expand All @@ -32,6 +30,10 @@ func main() {
will be mapped to the ini section "[section_name]" and the key
"KEY_NAME" with the value as provided.

Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME__FILE"
will be mapped to the ini section "[section_name]" and the key
"KEY_NAME" with the value loaded from the specified file.

Environment variables are usually restricted to a reduced character
set "0-9A-Z_" - in order to allow the setting of sections with
characters outside of that set, they should be escaped as following:
Expand Down Expand Up @@ -96,11 +98,11 @@ func runEnvironmentToIni(c *cli.Context) error {
setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)

cfg := ini.Empty()
isFile, err := util.IsFile(setting.CustomConf)
confFileExists, err := util.IsFile(setting.CustomConf)
if err != nil {
log.Fatal("Unable to check if %s is a file. Error: %v", setting.CustomConf, err)
}
if isFile {
if confFileExists {
if err := cfg.Append(setting.CustomConf); err != nil {
log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
}
Expand All @@ -109,47 +111,11 @@ func runEnvironmentToIni(c *cli.Context) error {
}
cfg.NameMapper = ini.SnackCase

changed := false
prefixGitea := c.String("prefix") + "__"
suffixFile := "__FILE"
changed := setting.EnvironmentToConfig(cfg, prefixGitea, suffixFile, os.Environ())

prefix := c.String("prefix") + "__"

for _, kv := range os.Environ() {
idx := strings.IndexByte(kv, '=')
if idx < 0 {
continue
}
eKey := kv[:idx]
value := kv[idx+1:]
if !strings.HasPrefix(eKey, prefix) {
continue
}
eKey = eKey[len(prefix):]
sectionName, keyName := DecodeSectionKey(eKey)
if len(keyName) == 0 {
continue
}
section, err := cfg.GetSection(sectionName)
if err != nil {
section, err = cfg.NewSection(sectionName)
if err != nil {
log.Error("Error creating section: %s : %v", sectionName, err)
continue
}
}
key := section.Key(keyName)
if key == nil {
key, err = section.NewKey(keyName, value)
if err != nil {
log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, value, err)
continue
}
}
oldValue := key.Value()
if !changed && oldValue != value {
changed = true
}
key.SetValue(value)
}
// try to save the config file
destination := c.String("out")
if len(destination) == 0 {
destination = setting.CustomConf
Expand All @@ -161,76 +127,20 @@ func runEnvironmentToIni(c *cli.Context) error {
return err
}
}

// clear Gitea's specific environment variables if requested
if c.Bool("clear") {
for _, kv := range os.Environ() {
idx := strings.IndexByte(kv, '=')
if idx < 0 {
continue
}
eKey := kv[:idx]
if strings.HasPrefix(eKey, prefix) {
if strings.HasPrefix(eKey, prefixGitea) {
_ = os.Unsetenv(eKey)
}
}
}
return nil
}

const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"

var escapeRegex = regexp.MustCompile(escapeRegexpString)

// DecodeSectionKey will decode a portable string encoded Section__Key pair
// Portable strings are considered to be of the form [A-Z0-9_]*
// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
// Section and Key are separated by a plain '__'.
// The entire section can be encoded as a UTF8 byte string
func DecodeSectionKey(encoded string) (string, string) {
section := ""
key := ""

inKey := false
last := 0
escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
for _, unescapeIdx := range escapeStringIndices {
preceding := encoded[last:unescapeIdx[0]]
if !inKey {
if splitter := strings.Index(preceding, "__"); splitter > -1 {
section += preceding[:splitter]
inKey = true
key += preceding[splitter+2:]
} else {
section += preceding
}
} else {
key += preceding
}
toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
decodedBytes := make([]byte, len(toDecode)/2)
for i := 0; i < len(toDecode)/2; i++ {
// Can ignore error here as we know these should be hexadecimal from the regexp
byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
decodedBytes[i] = byte(byteInt)
}
if inKey {
key += string(decodedBytes)
} else {
section += string(decodedBytes)
}
last = unescapeIdx[1]
}
remaining := encoded[last:]
if !inKey {
if splitter := strings.Index(remaining, "__"); splitter > -1 {
section += remaining[:splitter]
key += remaining[splitter+2:]
} else {
section += remaining
}
} else {
key += remaining
}
section = strings.ToLower(section)
return section, key
return nil
}
15 changes: 12 additions & 3 deletions docs/content/doc/installation/with-docker-rootless.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -286,9 +286,18 @@ docker-compose up -d

## Managing Deployments With Environment Variables

In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini).

These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`:
In addition to the environment variables above, any settings in `app.ini` can be set
or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`.
These settings are applied each time the docker container starts.
Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini).

These environment variables can be passed to the docker container in `docker-compose.yml`.
The following example will enable a smtp mail server if the required env variables
`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host
or in a `.env` file in the same directory as `docker-compose.yml`.

The settings can be also set or overridden with the content of a file by defining an environment variable of the form:
`GITEA__section_name__KEY_NAME__FILE` that points to a file.

```bash
...
Expand Down
15 changes: 12 additions & 3 deletions docs/content/doc/installation/with-docker.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -287,9 +287,18 @@ docker-compose up -d

## Managing Deployments With Environment Variables

In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini).

These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`:
In addition to the environment variables above, any settings in `app.ini` can be set
or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`.
These settings are applied each time the docker container starts.
Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini).

These environment variables can be passed to the docker container in `docker-compose.yml`.
The following example will enable an smtp mail server if the required env variables
`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host
or in a `.env` file in the same directory as `docker-compose.yml`.

The settings can be also set or overridden with the content of a file by defining an environment variable of the form:
`GITEA__section_name__KEY_NAME__FILE` that points to a file.

```bash
...
Expand Down
142 changes: 142 additions & 0 deletions modules/setting/config_env.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package setting

import (
"os"
"regexp"
"strconv"
"strings"

"code.gitea.io/gitea/modules/log"

"gopkg.in/ini.v1"
)

const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_"

var escapeRegex = regexp.MustCompile(escapeRegexpString)

// decodeEnvSectionKey will decode a portable string encoded Section__Key pair
// Portable strings are considered to be of the form [A-Z0-9_]*
// We will encode a disallowed value as the UTF8 byte string preceded by _0X and
// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.'
// Section and Key are separated by a plain '__'.
// The entire section can be encoded as a UTF8 byte string
func decodeEnvSectionKey(encoded string) (ok bool, section, key string) {
inKey := false
last := 0
escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1)
for _, unescapeIdx := range escapeStringIndices {
preceding := encoded[last:unescapeIdx[0]]
if !inKey {
if splitter := strings.Index(preceding, "__"); splitter > -1 {
section += preceding[:splitter]
inKey = true
key += preceding[splitter+2:]
} else {
section += preceding
}
} else {
key += preceding
}
toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1]
decodedBytes := make([]byte, len(toDecode)/2)
for i := 0; i < len(toDecode)/2; i++ {
// Can ignore error here as we know these should be hexadecimal from the regexp
byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0)
decodedBytes[i] = byte(byteInt)
}
if inKey {
key += string(decodedBytes)
} else {
section += string(decodedBytes)
}
last = unescapeIdx[1]
}
remaining := encoded[last:]
if !inKey {
if splitter := strings.Index(remaining, "__"); splitter > -1 {
section += remaining[:splitter]
key += remaining[splitter+2:]
} else {
section += remaining
}
} else {
key += remaining
}
section = strings.ToLower(section)
ok = section != "" && key != ""
if !ok {
section = ""
key = ""
}
return ok, section, key
}

// decodeEnvironmentKey decode the environment key to section and key
// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE
func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) {
if !strings.HasPrefix(envKey, prefixGitea) {
return false, "", "", false
}
if strings.HasSuffix(envKey, suffixFile) {
useFileValue = true
envKey = envKey[:len(envKey)-len(suffixFile)]
}
ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):])
return ok, section, key, useFileValue
}

func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) {
for _, kv := range envs {
idx := strings.IndexByte(kv, '=')
if idx < 0 {
continue
}

// parse the environment variable to config section name and key name
envKey := kv[:idx]
envValue := kv[idx+1:]
ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey)
if !ok {
continue
}

// use environment value as config value, or read the file content as value if the key indicates a file
keyValue := envValue
if useFileValue {
fileContent, err := os.ReadFile(envValue)
if err != nil {
log.Error("Error reading file for %s : %v", envKey, envValue, err)
continue
}
keyValue = string(fileContent)
}

// try to set the config value if necessary
section, err := cfg.GetSection(sectionName)
if err != nil {
section, err = cfg.NewSection(sectionName)
if err != nil {
log.Error("Error creating section: %s : %v", sectionName, err)
continue
}
}
key := section.Key(keyName)
if key == nil {
key, err = section.NewKey(keyName, keyValue)
if err != nil {
log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err)
continue
}
}
oldValue := key.Value()
if !changed && oldValue != keyValue {
changed = true
}
key.SetValue(keyValue)
}
return changed
}
Loading