From 9178b36189b26898e3de525ed2978056772913e1 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Tue, 3 Oct 2023 00:56:06 +0200 Subject: [PATCH 1/6] Implement client library Signed-off-by: Knut Ahlers --- pkg/client/client.go | 199 +++++++++++++++++++++++++++++++++++++ pkg/client/client_test.go | 38 +++++++ pkg/client/go.mod | 15 +++ pkg/client/go.sum | 14 +++ pkg/client/otsMeta.go | 92 +++++++++++++++++ pkg/client/otsMeta_test.go | 75 ++++++++++++++ 6 files changed, 433 insertions(+) create mode 100644 pkg/client/client.go create mode 100644 pkg/client/client_test.go create mode 100644 pkg/client/go.mod create mode 100644 pkg/client/go.sum create mode 100644 pkg/client/otsMeta.go create mode 100644 pkg/client/otsMeta_test.go diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..2675b4b --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,199 @@ +// Package client implements a client library for OTS supporting the +// OTSMeta content format for file upload support +package client + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha512" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/Luzifer/go-openssl/v4" +) + +// HTTPClient defines the client to use for create and fetch requests +// and can be overwritten to provide authentication +var HTTPClient = http.DefaultClient + +// KeyDerivationFunc defines the key derivation algorithm used in OTS +// to derive the key / iv from the password for encryption. You only +// should change this if you are running an OTS instance with modified +// parameters. +// +// The corresponding settings are found in `/src/crypto.js` in the OTS +// source code. +var KeyDerivationFunc = openssl.NewPBKDF2Generator(sha512.New, 300000) //nolint:gomnd // that's the definition + +// PasswordLength defines the length of the generated encryption password +var PasswordLength = 20 + +// RequestTimeout defines how long the request to the OTS instance for +// create and fetch may take +var RequestTimeout = 5 * time.Second + +// UserAgent defines the user-agent to send when interacting with an +// OTS instance. When using this library please set this to something +// the operator of the instance can determine your client from and +// provide an URL to useful information about your tool. +var UserAgent = "ots-client/1.x +https://github.com/Luzifer/ots" + +// Create serializes the secret and creates a new secret on the +// instance given by its URL. +// +// The given URL should point to the frontend of the instance. Do not +// include the API paths, they are added automatically. For the +// expireIn parameter zero value can be used to use server-default. +// +// So for OTS.fyi you'd use `New("https://ots.fyi/")` +func Create(instanceURL string, secret Secret, expireIn time.Duration) (string, time.Time, error) { + u, err := url.Parse(instanceURL) + if err != nil { + return "", time.Time{}, fmt.Errorf("parsing instance URL: %w", err) + } + + pass, err := genPass() + if err != nil { + return "", time.Time{}, fmt.Errorf("generating password: %w", err) + } + + data, err := secret.serialize(pass) + if err != nil { + return "", time.Time{}, fmt.Errorf("serializing data: %w", err) + } + + body := new(bytes.Buffer) + if err = json.NewEncoder(body).Encode(struct { + Secret string `json:"secret"` + }{Secret: string(data)}); err != nil { + return "", time.Time{}, fmt.Errorf("encoding request payload: %w", err) + } + + createURL := u.JoinPath(strings.Join([]string{".", "api", "create"}, "/")) + ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) + defer cancel() + + if expireIn > time.Second { + createURL.RawQuery = url.Values{ + "expire": []string{strconv.Itoa(int(expireIn / time.Second))}, + }.Encode() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, createURL.String(), body) + if err != nil { + return "", time.Time{}, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", UserAgent) + + resp, err := HTTPClient.Do(req) + if err != nil { + return "", time.Time{}, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() //nolint:errcheck // possible leaked-fd, lib should not log, potential short-lived leak + + if resp.StatusCode != http.StatusCreated { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", time.Time{}, fmt.Errorf("unexpected HTTP status %d", resp.StatusCode) + } + return "", time.Time{}, fmt.Errorf("unexpected HTTP status %d (%s)", resp.StatusCode, respBody) + } + + var payload struct { + ExpiresAt time.Time `json:"expires_at"` + SecretID string `json:"secret_id"` + Success bool `json:"success"` + } + + if err = json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return "", time.Time{}, fmt.Errorf("decoding response: %w", err) + } + + u.Fragment = strings.Join([]string{payload.SecretID, pass}, "|") + + return u.String(), payload.ExpiresAt, nil +} + +// Fetch retrieves a secret by its given URL. The URL given must +// include the fragment (part after the `#`) with the secret ID and +// the encryption passphrase. +// +// The object returned will always be an OTSMeta object even in case +// the secret is a plain secret without attachments. +func Fetch(secretURL string) (s Secret, err error) { + u, err := url.Parse(secretURL) + if err != nil { + return s, fmt.Errorf("parsing secret URL: %w", err) + } + + fragment, err := url.QueryUnescape(u.Fragment) + if err != nil { + return s, fmt.Errorf("unescaping fragment: %w", err) + } + fragmentParts := strings.SplitN(fragment, "|", 2) //nolint:gomnd + + fetchURL := u.JoinPath(strings.Join([]string{".", "api", "get", fragmentParts[0]}, "/")).String() + ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL, nil) + if err != nil { + return s, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("User-Agent", UserAgent) + + resp, err := HTTPClient.Do(req) + if err != nil { + return s, fmt.Errorf("executing request: %w", err) + } + defer resp.Body.Close() //nolint:errcheck // possible leaked-fd, lib should not log, potential short-lived leak + + if resp.StatusCode != http.StatusOK { + return s, fmt.Errorf("unexpected HTTP status %d", resp.StatusCode) + } + + var payload struct { + Secret string `json:"secret"` + } + + if err = json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return s, fmt.Errorf("decoding response body: %w", err) + } + + if err = s.read([]byte(payload.Secret), fragmentParts[1]); err != nil { + return s, fmt.Errorf("decoding secret: %w", err) + } + + return s, nil +} + +func genPass() (string, error) { + var ( + charSet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + pass = make([]byte, PasswordLength) + + n int + err error + ) + + for n < PasswordLength { + n, err = rand.Read(pass) + if err != nil { + return "", fmt.Errorf("reading random data: %w", err) + } + } + + for i := 0; i < PasswordLength; i++ { + pass[i] = charSet[int(pass[i])%len(charSet)] + } + + return string(pass), nil +} diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go new file mode 100644 index 0000000..996fc58 --- /dev/null +++ b/pkg/client/client_test.go @@ -0,0 +1,38 @@ +package client + +import ( + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGeneratePassword(t *testing.T) { + pass, err := genPass() + require.NoError(t, err) + + assert.Len(t, pass, PasswordLength) + assert.Regexp(t, regexp.MustCompile(`^[0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ]+$`), pass) +} + +func TestIntegration(t *testing.T) { + s := Secret{ + Secret: "I'm a secret!", + Attachments: []SecretAttachment{{ + Name: "secret.txt", + Type: "text/plain", + Content: []byte("I'm a very secret file.\n"), + }}, + } + + secretURL, _, err := Create("https://ots.fyi/", s, time.Minute) + require.NoError(t, err) + assert.Regexp(t, regexp.MustCompile(`^https://ots.fyi/#[0-9a-f-]+%7C[0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ]+$`), secretURL) + + apiSecret, err := Fetch(secretURL) + require.NoError(t, err) + + assert.Equal(t, s, apiSecret) +} diff --git a/pkg/client/go.mod b/pkg/client/go.mod new file mode 100644 index 0000000..256fda3 --- /dev/null +++ b/pkg/client/go.mod @@ -0,0 +1,15 @@ +module github.com/Luzifer/ots/pkg/client + +go 1.21.1 + +require ( + github.com/Luzifer/go-openssl/v4 v4.2.1 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/pkg/client/go.sum b/pkg/client/go.sum new file mode 100644 index 0000000..9e1f53d --- /dev/null +++ b/pkg/client/go.sum @@ -0,0 +1,14 @@ +github.com/Luzifer/go-openssl/v4 v4.2.1 h1:0+/gaQ5TcBhGmVqGrfyA21eujlbbaNwj0VlOA3nh4ts= +github.com/Luzifer/go-openssl/v4 v4.2.1/go.mod h1:CZZZWY0buCtkxrkqDPQYigC4Kn55UuO97TEoV+hwz2s= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/client/otsMeta.go b/pkg/client/otsMeta.go new file mode 100644 index 0000000..fbedd21 --- /dev/null +++ b/pkg/client/otsMeta.go @@ -0,0 +1,92 @@ +package client + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/Luzifer/go-openssl/v4" +) + +var metaMarker = []byte("OTSMeta") + +type ( + // Secret represents a secret parsed from / prepared for + // serialization to the OTS API + Secret struct { + Secret string `json:"secret"` + Attachments []SecretAttachment `json:"attachments,omitempty"` + } + + // SecretAttachment represents a file attached to a Secret. The Data + // property must be the plain content (binary / text / ...) of the + // file to attach. The base64 en-/decoding is done transparently. + // The Name is the name of the file shown to the user (so ideally + // should be the file-name on the source system). The Type should + // contain the mime time of the file or an empty string. + SecretAttachment struct { + Name string `json:"name"` + Type string `json:"type"` + Data string `json:"data"` + Content []byte `json:"-"` + } +) + +func (o *Secret) read(data []byte, passphrase string) (err error) { + if passphrase != "" { + if data, err = openssl.New().DecryptBytes(passphrase, data, KeyDerivationFunc); err != nil { + return fmt.Errorf("decrypting data: %w", err) + } + } + + if !bytes.HasPrefix(data, metaMarker) { + // We have a simple secret, makes less effort for us + o.Secret = string(data) + return nil + } + + if err = json.Unmarshal(data[len(metaMarker):], o); err != nil { + return fmt.Errorf("decoding JSON payload: %w", err) + } + + for i := range o.Attachments { + o.Attachments[i].Content, err = base64.StdEncoding.DecodeString(o.Attachments[i].Data) + if err != nil { + return fmt.Errorf("decoding attachment %d: %w", i, err) + } + } + + return nil +} + +func (o Secret) serialize(passphrase string) ([]byte, error) { + var data []byte + + if len(o.Attachments) == 0 { + // No attachments? No problem, we create a classic simple secret + data = []byte(o.Secret) + } else { + for i := range o.Attachments { + o.Attachments[i].Data = base64.StdEncoding.EncodeToString(o.Attachments[i].Content) + } + + j, err := json.Marshal(o) + if err != nil { + return nil, fmt.Errorf("encoding JSON payload: %w", err) + } + + data = append(metaMarker, j...) //nolint:gocritic // :shrug: + } + + if passphrase == "" { + // No encryption requested + return data, nil + } + + out, err := openssl.New().EncryptBytes(passphrase, data, KeyDerivationFunc) + if err != nil { + return nil, fmt.Errorf("encrypting data: %w", err) + } + return out, nil +} diff --git a/pkg/client/otsMeta_test.go b/pkg/client/otsMeta_test.go new file mode 100644 index 0000000..67995e2 --- /dev/null +++ b/pkg/client/otsMeta_test.go @@ -0,0 +1,75 @@ +package client + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadOTSMeta(t *testing.T) { + var ( + //#nosec:G101 // Hardcoded credentials, just test-data + secretData = "U2FsdGVkX1+7kNgAK57O/qdbsukK3OchMyMyE1tWzVJVlc9f9bkp8iaFHbwR7Q3b8tWhWmPAcfeOoBJH2zl1iNbIHWsmMKu3+pzE5wTE4wl31dOboV8LgsMChBFL5RQpda0iGku32BcB4tYEyb2VHcM/kkXNJh9lW1vRyiNx0iF8pe05JUkkmJJrnzIKC+/efZEfF2YX7fOaBC1+8AAhlg==" + //#nosec:G101 // Hardcoded credentials, just test-data + pass = "IKeiXsyGuVWdMUG8Fj3R" + s Secret + ) + + err := s.read([]byte(secretData), pass) + require.NoError(t, err) + + assert.Equal(t, Secret{ + Secret: "I'm a secret!", + Attachments: []SecretAttachment{{ + Name: "secret.txt", + Type: "text/plain", + Data: "SSdtIGEgdmVyeSBzZWNyZXQgZmlsZS4K", + Content: []byte("I'm a very secret file.\n"), + }}, + }, s) +} + +func TestReadSimpleSecret(t *testing.T) { + var ( + //#nosec:G101 // Hardcoded credentials, just test-data + secretData = "U2FsdGVkX18cvbYVRsD5cxMKKAHtMRmteu88tPwRtOk=" + //#nosec:G101 // Hardcoded credentials, just test-data + pass = "YQHdft6hDnp575olczeq" + s Secret + ) + + err := s.read([]byte(secretData), pass) + require.NoError(t, err) + + assert.Equal(t, Secret{ + Secret: "I'm a secret!", + }, s) +} + +func TestSerializeOTSMeta(t *testing.T) { + // NOTE(kahlers): We're using an empty passphrase here to achieve + // testability of the output. The data is not encrypted in this + // case. + data, err := Secret{ + Secret: "I'm a secret!", + Attachments: []SecretAttachment{{ + Name: "secret.txt", + Type: "text/plain", + Content: []byte("I'm a very secret file.\n"), + }}, + }.serialize("") + require.NoError(t, err) + + assert.Equal(t, []byte(`OTSMeta{"secret":"I'm a secret!","attachments":[{"name":"secret.txt","type":"text/plain","data":"SSdtIGEgdmVyeSBzZWNyZXQgZmlsZS4K"}]}`), data) +} + +func TestSerializeSimpleSecret(t *testing.T) { + // NOTE(kahlers): We're using an empty passphrase here to achieve + // testability of the output. The data is not encrypted in this + // case. + data, err := Secret{Secret: "I'm a secret!"}.serialize("") + require.NoError(t, err) + + assert.Equal(t, []byte("I'm a secret!"), data) +} From bbf80c852451ca90775b8b5da52dfd8c0871de02 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Tue, 3 Oct 2023 15:55:04 +0200 Subject: [PATCH 2/6] Implement ots-cli command Signed-off-by: Knut Ahlers --- cmd/ots-cli/cmd_create.go | 105 ++++++++++++++++++++++++++++++++++++++ cmd/ots-cli/cmd_fetch.go | 96 ++++++++++++++++++++++++++++++++++ cmd/ots-cli/cmd_root.go | 32 ++++++++++++ cmd/ots-cli/go.mod | 19 +++++++ cmd/ots-cli/go.sum | 30 +++++++++++ cmd/ots-cli/main.go | 9 ++++ 6 files changed, 291 insertions(+) create mode 100644 cmd/ots-cli/cmd_create.go create mode 100644 cmd/ots-cli/cmd_fetch.go create mode 100644 cmd/ots-cli/cmd_root.go create mode 100644 cmd/ots-cli/go.mod create mode 100644 cmd/ots-cli/go.sum create mode 100644 cmd/ots-cli/main.go diff --git a/cmd/ots-cli/cmd_create.go b/cmd/ots-cli/cmd_create.go new file mode 100644 index 0000000..e0f3018 --- /dev/null +++ b/cmd/ots-cli/cmd_create.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "io" + "mime" + "os" + "path" + + "github.com/Luzifer/ots/pkg/client" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var createCmd = &cobra.Command{ + Use: "create [-f file]... [--instance url] [--secret-from file]", + Short: "Create a new encrypted secret in the given OTS instance", + Long: "", + Example: `echo "I'm a very secret secret" | ots-cli create`, + Args: cobra.NoArgs, + RunE: createRunE, +} + +func init() { + createCmd.Flags().Duration("expire", 0, "When to expire the secret (0 to use server-default)") + createCmd.Flags().String("instance", "https://ots.fyi/", "Instance to create the secret with") + createCmd.Flags().StringSliceP("file", "f", nil, "File(s) to attach to the secret") + createCmd.Flags().String("secret-from", "-", `File to read the secret content from ("-" for STDIN)`) + rootCmd.AddCommand(createCmd) +} + +func createRunE(cmd *cobra.Command, _ []string) error { + var secret client.Secret + + // Read the secret content + logrus.Info("reading secret content...") + secretSourceName, err := cmd.Flags().GetString("secret-from") + if err != nil { + return fmt.Errorf("getting secret-from flag: %w", err) + } + + var secretSource io.Reader + if secretSourceName == "-" { + secretSource = os.Stdin + } else { + f, err := os.Open(secretSourceName) //#nosec:G304 // Opening user specified file is intended + if err != nil { + return fmt.Errorf("opening secret-from file: %w", err) + } + defer f.Close() //nolint:errcheck // The file will be force-closed by program exit + secretSource = f + } + + secretContent, err := io.ReadAll(secretSource) + if err != nil { + return fmt.Errorf("reading secret content: %w", err) + } + secret.Secret = string(secretContent) + + // Attach any file given + files, err := cmd.Flags().GetStringSlice("file") + if err != nil { + return fmt.Errorf("getting file flag: %w", err) + } + for _, f := range files { + logrus.WithField("file", f).Info("attaching file...") + content, err := os.ReadFile(f) //#nosec:G304 // Opening user specified file is intended + if err != nil { + return fmt.Errorf("reading attachment %q: %w", f, err) + } + + secret.Attachments = append(secret.Attachments, client.SecretAttachment{ + Name: f, + Type: mime.TypeByExtension(path.Ext(f)), + Content: content, + }) + } + + // Create the secret + logrus.Info("creating the secret...") + instanceURL, err := cmd.Flags().GetString("instance") + if err != nil { + return fmt.Errorf("getting instance flag: %w", err) + } + + expire, err := cmd.Flags().GetDuration("expire") + if err != nil { + return fmt.Errorf("getting expire flag: %w", err) + } + + secretURL, expiresAt, err := client.Create(instanceURL, secret, expire) + if err != nil { + return fmt.Errorf("creating secret: %w", err) + } + + // Tell them where to find the secret + if expiresAt.IsZero() { + logrus.Info("secret created, see URL below") + } else { + logrus.WithField("expires-at", expiresAt).Info("secret created, see URL below") + } + fmt.Println(secretURL) //nolint:forbidigo // Output intended for STDOUT + + return nil +} diff --git a/cmd/ots-cli/cmd_fetch.go b/cmd/ots-cli/cmd_fetch.go new file mode 100644 index 0000000..ce09228 --- /dev/null +++ b/cmd/ots-cli/cmd_fetch.go @@ -0,0 +1,96 @@ +package main + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path" + "strings" + + "github.com/Luzifer/ots/pkg/client" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +const storeFileMode = 0o600 // We assume the attached file to be a secret + +var fetchCmd = &cobra.Command{ + Use: "fetch url", + Short: "Retrieves a secret from the instance by its URL", + Long: "", + Args: cobra.ExactArgs(1), + RunE: fetchRunE, +} + +func init() { + fetchCmd.Flags().String("file-dir", ".", "Where to put files attached to the secret") + rootCmd.AddCommand(fetchCmd) +} + +func checkDirWritable(dir string) error { + tmpFile := path.Join(dir, ".ots-cli.tmp") + if err := os.WriteFile(tmpFile, []byte(""), storeFileMode); err != nil { + return fmt.Errorf("writing tmp-file: %w", err) + } + defer os.Remove(tmpFile) //nolint:errcheck // We don't really care + + return nil +} + +func fetchRunE(cmd *cobra.Command, args []string) error { + fileDir, err := cmd.Flags().GetString("file-dir") + if err != nil { + return fmt.Errorf("getting file-dir parameter: %w", err) + } + + // First lets check whether we potentially can write files + if err := checkDirWritable(fileDir); err != nil { + return fmt.Errorf("checking for directory write: %w", err) + } + + logrus.Info("fetching secret...") + secret, err := client.Fetch(args[0]) + if err != nil { + return fmt.Errorf("fetching secret") + } + + for _, f := range secret.Attachments { + logrus.WithField("file", f.Name).Info("storing file...") + if err = storeAttachment(fileDir, f); err != nil { + return fmt.Errorf("saving file to disk: %w", err) + } + } + + fmt.Println(secret.Secret) //nolint:forbidigo // Output intended for STDOUT + + return nil +} + +func storeAttachment(dir string, f client.SecretAttachment) error { + // First lets find a free file name to save the file as + var ( + fileNameFragments = strings.SplitN(f.Name, ".", 2) //nolint:gomnd + i int + storeName = path.Join(dir, f.Name) + storeNameTpl string + ) + + if len(fileNameFragments) == 1 { + storeNameTpl = fmt.Sprintf("%s (%%d)", fileNameFragments[0]) + } else { + storeNameTpl = fmt.Sprintf("%s (%%d).%s", fileNameFragments[0], fileNameFragments[1]) + } + + for _, err := os.Stat(storeName); !errors.Is(err, fs.ErrNotExist); _, err = os.Stat(storeName) { + i++ + storeName = fmt.Sprintf(storeNameTpl, i) + } + + // So we finally found a filename we can use + if err := os.WriteFile(storeName, f.Content, storeFileMode); err != nil { + return fmt.Errorf("writing file: %w", err) + } + + return nil +} diff --git a/cmd/ots-cli/cmd_root.go b/cmd/ots-cli/cmd_root.go new file mode 100644 index 0000000..48928f8 --- /dev/null +++ b/cmd/ots-cli/cmd_root.go @@ -0,0 +1,32 @@ +package main + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Short: "Utility to interact with encrypted secrets in an OTS instance", + PersistentPreRunE: rootPersistentPreRunE, +} + +func init() { + rootCmd.PersistentFlags().String("log-level", "info", "Level to use for logging (trace, debug, info, warn, error, fatal)") +} + +func rootPersistentPreRunE(cmd *cobra.Command, _ []string) error { + sll, err := cmd.Flags().GetString("log-level") + if err != nil { + return fmt.Errorf("getting log-level: %w", err) + } + + ll, err := logrus.ParseLevel(sll) + if err != nil { + return fmt.Errorf("parsing log-level: %w", err) + } + logrus.SetLevel(ll) + + return nil +} diff --git a/cmd/ots-cli/go.mod b/cmd/ots-cli/go.mod new file mode 100644 index 0000000..729ee15 --- /dev/null +++ b/cmd/ots-cli/go.mod @@ -0,0 +1,19 @@ +module github.com/Luzifer/ots/cmd/ots-cli + +go 1.21.1 + +replace github.com/Luzifer/ots/pkg/client => ../../pkg/client + +require ( + github.com/Luzifer/ots/pkg/client v0.0.0-00010101000000-000000000000 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.7.0 +) + +require ( + github.com/Luzifer/go-openssl/v4 v4.2.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/sys v0.11.0 // indirect +) diff --git a/cmd/ots-cli/go.sum b/cmd/ots-cli/go.sum new file mode 100644 index 0000000..0e18c54 --- /dev/null +++ b/cmd/ots-cli/go.sum @@ -0,0 +1,30 @@ +github.com/Luzifer/go-openssl/v4 v4.2.1 h1:0+/gaQ5TcBhGmVqGrfyA21eujlbbaNwj0VlOA3nh4ts= +github.com/Luzifer/go-openssl/v4 v4.2.1/go.mod h1:CZZZWY0buCtkxrkqDPQYigC4Kn55UuO97TEoV+hwz2s= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/ots-cli/main.go b/cmd/ots-cli/main.go new file mode 100644 index 0000000..2e7006e --- /dev/null +++ b/cmd/ots-cli/main.go @@ -0,0 +1,9 @@ +package main + +import "os" + +func main() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} From c02a294ede00975a2647fccfb7e3a6a7462dd33b Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Wed, 4 Oct 2023 18:31:42 +0200 Subject: [PATCH 3/6] CI: Test all packages Signed-off-by: Knut Ahlers --- .github/workflows/test-and-build.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 27f4d2a..a50fcdb 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -50,9 +50,19 @@ jobs: - name: Marking workdir safe run: git config --global --add safe.directory /__w/ots/ots - - name: Lint and test code + - name: 'Lint and test code: API' run: | - go test -v ./... + go test -cover -v ./... + + - name: 'Lint and test code: Client' + working-directory: ./pkg/client + run: | + go test -cover -v ./... + + - name: 'Lint and test code: OTS-CLI' + working-directory: ./cmd/ots-cli + run: | + go test -cover -v ./... - name: Generate (and validate) translations run: make translate From 04e9cda1d185ed966a502e1434b5622dbb1b2405 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Wed, 4 Oct 2023 19:14:38 +0200 Subject: [PATCH 4/6] CI: Add custom build-script containing API and CLI Signed-off-by: Knut Ahlers --- .gitignore | 1 + Makefile | 3 +-- ci/build.sh | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 ci/build.sh diff --git a/.gitignore b/.gitignore index 794aad4..48dace5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.build customize.yaml frontend/api.html frontend/app.css diff --git a/Makefile b/Makefile index 69bf56e..f5d80eb 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,7 @@ generate-inner: node ./ci/build.mjs publish: download_libs generate-inner generate-apidocs - curl -sSLo golang.sh https://raw.githubusercontent.com/Luzifer/github-publish/master/golang.sh - bash golang.sh + bash ./ci/build.sh translate: cd ci/translate && go run . diff --git a/ci/build.sh b/ci/build.sh new file mode 100644 index 0000000..fe0bf38 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +osarch=( + darwin/amd64 + darwin/arm64 + linux/amd64 + linux/arm + linux/arm64 + windows/amd64 +) + +function go_package() { + cd "${4}" + + local outname="${3}" + [[ $1 == windows ]] && outname="${3}.exe" + + log "=> Building ${3} for ${1}/${2}..." + CGO_ENABLED=0 GOARCH=$2 GOOS=$1 go build \ + -ldflags "-s -w -X main.version=${version}" \ + -mod=readonly \ + -trimpath \ + -o "${outname}" + + if [[ $1 == linux ]]; then + log "=> Packging ${3} as ${3}_${1}_${2}.tgz..." + tar -czf "${builddir}/${3}_${1}_${2}.tgz" "${outname}" + else + log "=> Packging ${3} as ${3}_${1}_${2}.zip..." + zip "${builddir}/${3}_${1}_${2}.zip" "${outname}" + fi + + rm "${outname}" +} + +function go_package_all() { + for oa in "${osarch[@]}"; do + local os=$(cut -d / -f 1 <<<"${oa}") + local arch=$(cut -d / -f 2 <<<"${oa}") + (go_package "${os}" "${arch}" "${1}" "${2}") + done +} + +function log() { + echo "[$(date +%H:%M:%S)] $@" >&2 +} + +root=$(pwd) +builddir="${root}/.build" +version="$(git describe --tags --always || echo dev)" + +log "Building version ${version}..." + +log "Resetting output directory..." +rm -rf "${builddir}" +mkdir -p "${builddir}" + +log "Building API-Server..." +go_package_all "ots" "." + +log "Building OTS-CLI..." +go_package_all "ots-cli" "./cmd/ots-cli" + +log "Generating SHA256SUMS file..." +(cd "${builddir}" && sha256sum * | tee SHA256SUMS) From 366eeb0b5035879984c7b0623c41eb527a1b7612 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Wed, 4 Oct 2023 22:00:27 +0200 Subject: [PATCH 5/6] Add format documentation for the OTSMeta format Signed-off-by: Knut Ahlers --- cmd/ots-cli/.gitignore | 1 + docs/OTSMeta-format.md | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 cmd/ots-cli/.gitignore create mode 100644 docs/OTSMeta-format.md diff --git a/cmd/ots-cli/.gitignore b/cmd/ots-cli/.gitignore new file mode 100644 index 0000000..69cec87 --- /dev/null +++ b/cmd/ots-cli/.gitignore @@ -0,0 +1 @@ +ots-cli diff --git a/docs/OTSMeta-format.md b/docs/OTSMeta-format.md new file mode 100644 index 0000000..a9508c2 --- /dev/null +++ b/docs/OTSMeta-format.md @@ -0,0 +1,56 @@ +> OTS uses two different formats to store secrets under the hood. Both of them can be read and written by the frontend implementation as well as by the `ots-cli` application. + +## Simple Format + +The simple format is the format used by OTS since day one and is the, well, most simple one. It only consists of the secret encrypted using OpenSSL AES-256-CBC compatible encryption. This format is preferred for backwards compatibility when no other reasons require the use of the OTS-Meta format. + +```console +# openssl enc -aes-256-cbc -pbkdf2 -md sha512 -iter 300000 -pass pass:12345678 -a -A <<<"I'm a secret" +U2FsdGVkX19G3GuIw3LGM0PVQmavPU/LnWvJhcLeYvs= + +# curl -H 'Content-Type: application/json' -d '{"secret": "U2FsdGVkX19G3GuIw3LGM0PVQmavPU/LnWvJhcLeYvs="}' https://ots.fyi/api/create +{"success":true,"expires_at":"2023-10-11T19:45:01.315587714Z","secret_id":"bbd53ec5-8ee9-4df5-a630-9561313a348a"} + +# ots-cli fetch "https://ots.fyi/#bbd53ec5-8ee9-4df5-a630-9561313a348a%7C12345678" +INFO[0000] fetching secret... +I'm a secret +``` + +## OTSMeta Format + +The OTSMeta format was first introduced in `v1.9.0` of OTS together with the possibility to attach files to the secret. It contains structured data with a banner to differentiate between a simple JSON shared through OTS and the OTSMeta format. The OTSMeta structure itself is a simple JSON document containing a secret and a number of attachments having their contents base64 encoded: + +```json +{ + "secret": "I'm a secret", + "attachments": [ + { + "name": "file.txt", + "type": "text/plain", + "data": "SSdtIGZpbGUgY29udGVudAo=" + } + ] +} +``` + +This structure is prefixed with the Banner `OTSMeta` and then shared the same way as a simple secret would be: + +```console +# ots-cli create -f file.txt <<<"I'm a secret" +INFO[0000] reading secret content... +INFO[0000] attaching file... file=file.txt +INFO[0000] creating the secret... +INFO[0000] secret created, see URL below expires-at="2023-10-11 19:52:30.816059504 +0000 UTC" +https://ots.fyi/#6a6be08c-97d7-4970-a202-5bb6964460d8%7CwNUURZ0LRrQAhaczdZfj + +# curl -sS https://ots.fyi/api/get/6a6be08c-97d7-4970-a202-5bb6964460d8 | jq -r .secret >/tmp/secret.bin +# openssl enc -aes-256-cbc -pbkdf2 -md sha512 -iter 300000 -pass pass:wNUURZ0LRrQAhaczdZfj -a -A -d Date: Wed, 4 Oct 2023 22:22:22 +0200 Subject: [PATCH 6/6] Remove pure-shell examples in favor of ots-cli Signed-off-by: Knut Ahlers --- cli_create.sh | 38 -------------------------------------- cli_get.sh | 28 ---------------------------- 2 files changed, 66 deletions(-) delete mode 100644 cli_create.sh delete mode 100644 cli_get.sh diff --git a/cli_create.sh b/cli_create.sh deleted file mode 100644 index 736e471..0000000 --- a/cli_create.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: ${INSTANCE:=https://ots.fyi} # Where to reach the API of the instance (omit trailing slash) - -deps=(curl jq) -for cmd in "${deps[@]}"; do - which ${cmd} >/dev/null || { - echo "'${cmd}' util is required for this script" - exit 1 - } -done - -# Get secret from CLI argument -SECRET=${1:-} -[[ -n $SECRET ]] || { - echo "Usage: $0 'secret to share'" - exit 1 -} - -# Generate a random 20 character password -pass=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 20 || true) - -# Encrypt the secret -ciphertext=$(echo "${SECRET}" | openssl aes-256-cbc -base64 -A -pass "pass:${pass}" -iter 300000 -md sha512 2>/dev/null) - -# Create a secret and extract the secret ID -id=$( - curl -sSf \ - -X POST \ - -H 'content-type: application/json' \ - -d "$(jq --arg secret "${ciphertext}" -cn '{"secret": $secret}')" \ - "${INSTANCE}/api/create" | - jq -r '.secret_id' -) - -# Display URL to user -echo -e "Secret is now available at:\n${INSTANCE}/#${id}%7C${pass}" diff --git a/cli_get.sh b/cli_get.sh deleted file mode 100644 index 96208fc..0000000 --- a/cli_get.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -euo pipefail - -deps=(curl jq) -for cmd in "${deps[@]}"; do - which ${cmd} >/dev/null || { - echo "'${cmd}' util is required for this script" - exit 1 - } -done - -# Get URL from CLI argument -url="${1:-}" -[[ -n $url ]] || { - echo "Usage: $0 'URL to get the secret'" - exit 1 -} -# normalize url and extract parts -url="${url/|/%7C}" -host="${url%%/\#*}" -idpass="${url##*\#}" -pass="${idpass##*\%7C}" -id="${idpass%%\%7C*}" -geturl="${host}/api/get/${id}" - -# fetch secret and decrypt to STDOUT -curl -sSf "${geturl}" | jq -r ".secret" | - openssl aes-256-cbc -base64 -A -pass "pass:${pass}" -iter 300000 -md sha512 -d