feat(value): hash / compression / encoding stream transformers.

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ### Not released yet
 
+FEATURES:
+
+* sdk/value:
+  * `encoding` reader / writer factory. [#117](github.com/elastic/harp/pull/117)
+  * `compression` reader/writer factory. [#117](github.com/elastic/harp/pull/117)
+  * `hash` writer factory. [#117](github.com/elastic/harp/pull/117)
+
+* cli/transform:
+  * `compress`/`decompress` commands for various algorithms [#117](github.com/elastic/harp/pull/117)
+  * `hash`/`multihash` command for various hashing algorithms. [#117](github.com/elastic/harp/pull/117)
+  * `encode`/`decode` command for various encoding strategies [#117](github.com/elastic/harp/pull/117)
+
 ## 0.2.7
 
 ### 2022-02-14

FEATURES.md

@@ -74,7 +74,7 @@ and reproductible.
   * Generate
     * `BundleTemplate` for secret bootstrap
 
-* Builtin Ouput(s)
+* Builtin output(s)
   * `Harp Secret Container`
   * `Hashicorp Vault`
 

cmd/harp/internal/cmd/transform.go

@@ -39,6 +39,10 @@ var transformCmd = func() *cobra.Command {
 	cmd.AddCommand(transformVerifyCmd())
 	cmd.AddCommand(transformDecodeCmd())
 	cmd.AddCommand(transformEncodeCmd())
+	cmd.AddCommand(transformHashCmd())
+	cmd.AddCommand(transformCompressCmd())
+	cmd.AddCommand(transformDecompressCmd())
+	cmd.AddCommand(transformMultihashCmd())
 
 	return cmd
 }

cmd/harp/internal/cmd/transform_compress.go

@@ -0,0 +1,112 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"io"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/elastic/harp/pkg/sdk/cmdutil"
+	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/sdk/value/compression"
+)
+
+// -----------------------------------------------------------------------------
+
+type transformCompressParams struct {
+	inputPath  string
+	outputPath string
+	algorithm  string
+}
+
+var transformCompressCmd = func() *cobra.Command {
+	params := &transformCompressParams{}
+
+	longDesc := cmdutil.LongDesc(`
+	Compress the given input stream using the selected compression algorithm.
+
+	Supported compression:
+	  * identity - returns the unmodified input
+	  * gzip
+	  * lzw/lzw-msb/lzw-lsb
+	  * lz4
+	  * s2/snappy
+	  * zlib
+	  * flate/deflate
+	  * lzma
+	  * zstd`)
+
+	examples := cmdutil.Examples(`
+		# Compress a file
+		harp transform compress --in README.md --out README.md.gz --algorithm gzip
+
+		# Compress to STDOUT
+		harp transform compress --in README.md --algorithm gzip
+
+		# Compress from STDIN
+		harp transform compress --algorithm gzip`)
+
+	cmd := &cobra.Command{
+		Use:     "compress",
+		Short:   "Compress given input",
+		Long:    longDesc,
+		Example: examples,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Initialize logger and context
+			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-transform-compress", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
+			defer cancel()
+
+			// Read input
+			reader, err := cmdutil.Reader(params.inputPath)
+			if err != nil {
+				log.For(ctx).Fatal("unable to initialize input reader", zap.Error(err))
+			}
+
+			// Output writer
+			writer, err := cmdutil.Writer(params.outputPath)
+			if err != nil {
+				log.For(ctx).Fatal("unable to initialize output writer", zap.Error(err))
+			}
+
+			// Prepare compressor
+			compressedWriter, err := compression.NewWriter(writer, params.algorithm)
+			if err != nil {
+				log.SafeClose(compressedWriter, "unable to close the compression writer")
+				log.For(ctx).Fatal("unable to write encoded content", zap.Error(err))
+			}
+
+			// Process input as a stream.
+			if _, err := io.Copy(compressedWriter, reader); err != nil {
+				log.SafeClose(compressedWriter, "unable to close the compression writer")
+				log.For(ctx).Fatal("unable to process input", zap.Error(err))
+			}
+
+			// Close the writer
+			log.SafeClose(compressedWriter, "unable to close the compression writer")
+		},
+	}
+
+	// Parameters
+	cmd.Flags().StringVar(&params.inputPath, "in", "-", "Input path ('-' for stdin or filename)")
+	cmd.Flags().StringVar(&params.outputPath, "out", "-", "Output path ('-' for stdout or filename)")
+	cmd.Flags().StringVar(&params.algorithm, "algorithm", "gzip", "Compression algorithm")
+
+	return cmd
+}

cmd/harp/internal/cmd/transform_decode.go

@@ -18,15 +18,14 @@
 package cmd
 
 import (
-	"encoding/base64"
-	"encoding/hex"
 	"io"
 
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
 
 	"github.com/elastic/harp/pkg/sdk/cmdutil"
 	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/sdk/value/encoding"
 )
 
 // -----------------------------------------------------------------------------
@@ -40,9 +39,32 @@ type transformDecodeParams struct {
 var transformDecodeCmd = func() *cobra.Command {
 	params := &transformDecodeParams{}
 
+	longDesc := cmdutil.LongDesc(`
+	Decode the given input stream using the selected decoding strategy.
+
+	Supported codecs:
+	  * identity - returns the unmodified input
+	  * hex/base16 - returns the hexadecimal encoded input
+	  * base32 - returns the Base32 encoded input
+	  * base32hex - returns the Base32 with extended alphabet encoded input
+	  * base64 - returns the Base64 encoded input
+	  * base64raw - returns the Base64 encoded input without "=" padding
+	  * base64url - returns the Base64 encoded input using URL safe characters
+	  * base64urlraw - returns the Base64 encoded input using URL safe characters without "=" padding
+	  * base85 - returns the Base85 encoded input`)
+
+	examples := cmdutil.Examples(`
+		# Decode base64 from stdin
+		echo "dGVzdAo=" | harp transform decode --encoding base64
+
+		# Decode base64url from a file
+		harp transform decode --in test.txt --encoding base64url`)
+
 	cmd := &cobra.Command{
-		Use:   "decode",
-		Short: "Decode given input",
+		Use:     "decode",
+		Short:   "Decode given input",
+		Long:    longDesc,
+		Example: examples,
 		Run: func(cmd *cobra.Command, args []string) {
 			// Initialize logger and context
 			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-transform-decode", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
@@ -54,53 +76,29 @@ var transformDecodeCmd = func() *cobra.Command {
 				log.For(ctx).Fatal("unable to initialize input reader", zap.Error(err))
 			}
 
-			// Read input
+			// Output writer
 			writer, err := cmdutil.Writer(params.outputPath)
 			if err != nil {
 				log.For(ctx).Fatal("unable to initialize output writer", zap.Error(err))
 			}
 
-			// Drain reader
-			content, err := io.ReadAll(reader)
+			// Read and decode
+			out, err := encoding.NewReader(reader, params.encoding)
 			if err != nil {
-				log.For(ctx).Fatal("unable to drain input reader", zap.Error(err))
-			}
-
-			var (
-				out       []byte
-				errDecode error
-			)
-			// Apply transformation
-			switch params.encoding {
-			case "identity":
-				out = content
-			case "hex":
-				out, errDecode = hex.DecodeString(string(content))
-			case "base64":
-				out, errDecode = base64.StdEncoding.DecodeString(string(content))
-			case "base64raw":
-				out, errDecode = base64.RawStdEncoding.DecodeString(string(content))
-			case "base64url":
-				out, errDecode = base64.URLEncoding.DecodeString(string(content))
-			case "base64urlraw":
-				out, errDecode = base64.RawURLEncoding.DecodeString(string(content))
-			default:
-				log.For(ctx).Fatal("unhandled decoding strategy", zap.String("encoding", params.encoding))
-			}
-			if errDecode != nil {
-				log.For(ctx).Fatal("unable to decode input", zap.Error(err))
+				log.For(ctx).Fatal("unable to prepare input decoder", zap.Error(err))
 			}
 
-			if _, err = writer.Write(out); err != nil {
-				log.For(ctx).Fatal("unable to write result to writer", zap.Error(err))
+			// Process input as a stream.
+			if _, err := io.Copy(writer, out); err != nil {
+				log.For(ctx).Fatal("unable to process input", zap.Error(err))
 			}
 		},
 	}
 
 	// Parameters
 	cmd.Flags().StringVar(&params.inputPath, "in", "-", "Input path ('-' for stdin or filename)")
-	cmd.Flags().StringVar(&params.outputPath, "out", "-", "Output path ('-' for stdin or filename)")
-	cmd.Flags().StringVar(&params.encoding, "encoding", "identity", "Encoding strategy (hex, base64, base64raw, base64url, base64urlraw)")
+	cmd.Flags().StringVar(&params.outputPath, "out", "-", "Output path ('-' for stdout or filename)")
+	cmd.Flags().StringVar(&params.encoding, "encoding", "identity", "Encoding strategy")
 
 	return cmd
 }

cmd/harp/internal/cmd/transform_decompress.go

@@ -0,0 +1,112 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"io"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/elastic/harp/pkg/sdk/cmdutil"
+	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/sdk/value/compression"
+)
+
+// -----------------------------------------------------------------------------
+
+type transformDecompressParams struct {
+	inputPath  string
+	outputPath string
+	algorithm  string
+}
+
+var transformDecompressCmd = func() *cobra.Command {
+	params := &transformDecompressParams{}
+
+	longDesc := cmdutil.LongDesc(`
+	Decompress the given input stream using the selected compression algorithm.
+
+	Supported compression:
+	  * identity - returns the unmodified input
+	  * gzip
+	  * lzw/lzw-msb/lzw-lsb
+	  * lz4
+	  * s2/snappy
+	  * zlib
+	  * flate/deflate
+	  * lzma
+	  * zstd`)
+
+	examples := cmdutil.Examples(`
+	# Compress a file
+	harp transform decompress --in README.md.gz --out README.md --algorithm gzip
+
+	# Decompress to STDOUT
+	harp transform compress --in README.md.gz --algorithm gzip
+
+	# Decompress from STDIN
+	harp transform compress --algorithm gzip`)
+
+	cmd := &cobra.Command{
+		Use:     "decompress",
+		Short:   "Decompress given input",
+		Long:    longDesc,
+		Example: examples,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Initialize logger and context
+			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-transform-decompress", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
+			defer cancel()
+
+			// Read input
+			reader, err := cmdutil.Reader(params.inputPath)
+			if err != nil {
+				log.For(ctx).Fatal("unable to initialize input reader", zap.Error(err))
+			}
+
+			// Output writer
+			writer, err := cmdutil.Writer(params.outputPath)
+			if err != nil {
+				log.For(ctx).Fatal("unable to initialize output writer", zap.Error(err))
+			}
+
+			// Prepare compressor
+			compressedReader, err := compression.NewReader(reader, params.algorithm)
+			if err != nil {
+				log.SafeClose(compressedReader, "unable to close the compression writer")
+				log.For(ctx).Fatal("unable to write encoded content", zap.Error(err))
+			}
+
+			// Process input as a stream.
+			if _, err := io.Copy(writer, compressedReader); err != nil {
+				log.SafeClose(compressedReader, "unable to close the compression writer")
+				log.For(ctx).Fatal("unable to process input", zap.Error(err))
+			}
+
+			// Close the writer
+			log.SafeClose(compressedReader, "unable to close the compression writer")
+		},
+	}
+
+	// Parameters
+	cmd.Flags().StringVar(&params.inputPath, "in", "-", "Input path ('-' for stdin or filename)")
+	cmd.Flags().StringVar(&params.outputPath, "out", "-", "Output path ('-' for stdout or filename)")
+	cmd.Flags().StringVar(&params.algorithm, "algorithm", "gzip", "Compression algorithm")
+
+	return cmd
+}