Bar & histo logscale (#98)

Add graph-scaling support to bargraphs and histograms

cmd/bargraph.go

@@ -16,14 +16,21 @@ go run . bars -sz -m "\[(.+?)\].*\" (\d+)" -e "{$ {buckettime {1} year nginx} {2
 
 func bargraphFunction(c *cli.Context) error {
 	var (
-		stacked  = c.Bool("stacked")
-		sortName = c.String(helpers.DefaultSortFlag.Name)
+		stacked   = c.Bool("stacked")
+		sortName  = c.String(helpers.DefaultSortFlag.Name)
+		scaleName = c.String(helpers.ScaleFlag.Name)
 	)
 
 	vt := helpers.BuildVTermFromArguments(c)
 	counter := aggregation.NewSubKeyCounter()
 	writer := termrenderers.NewBarGraph(vt)
 	writer.Stacked = stacked
+	if c.IsSet(helpers.ScaleFlag.Name) {
+		if stacked {
+			return cli.Exit("Unable to set graph scale on stacked graphs", helpers.ExitCodeInvalidUsage)
+		}
+		writer.Scaler = helpers.BuildScalerOrFail(scaleName)
+	}
 
 	batcher := helpers.BuildBatcherFromArguments(c)
 	ext := helpers.BuildExtractorFromArguments(c, batcher)
@@ -72,6 +79,7 @@ func bargraphCommand() *cli.Command {
 			helpers.SnapshotFlag,
 			helpers.NoOutFlag,
 			helpers.CSVFlag,
+			helpers.ScaleFlag,
 		},
 	})
 }

cmd/bargraph_test.go

@@ -1,10 +1,20 @@
 package cmd
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
 
 func TestBarGraph(t *testing.T) {
 	testCommandSet(t, bargraphCommand(),
 		`-m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`,
 		`-o - -m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`,
+		`-o - -m "(.+) (\d+)" -e "{$ {1} {2}}" --scale log10 testdata/graph.txt`,
 	)
 }
+
+func TestBarGraphCantScaleAndStack(t *testing.T) {
+	err := testCommand(bargraphCommand(), "--stacked --scale log10 testdata/graph.txt")
+	assert.Error(t, err)
+}

cmd/histo.go

@@ -29,18 +29,20 @@ func writeHistoOutput(writer *termrenderers.HistoWriter, counter *aggregation.Ma
 
 func histoFunction(c *cli.Context) error {
 	var (
-		topItems = c.Int("n")
-		atLeast  = c.Int64("atleast")
-		extra    = c.Bool("extra")
-		all      = c.Bool("all")
-		sortName = c.String(helpers.DefaultSortFlag.Name)
+		topItems   = c.Int("n")
+		atLeast    = c.Int64("atleast")
+		extra      = c.Bool("extra")
+		all        = c.Bool("all")
+		sortName   = c.String(helpers.DefaultSortFlag.Name)
+		scalerName = c.String(helpers.ScaleFlag.Name)
 	)
 
 	vt := helpers.BuildVTermFromArguments(c)
 	counter := aggregation.NewCounter()
 	writer := termrenderers.NewHistogram(vt, topItems)
 	writer.ShowBar = c.Bool("bars") || extra
 	writer.ShowPercentage = c.Bool("percentage") || extra
+	writer.Scaler = helpers.BuildScalerOrFail(scalerName)
 
 	batcher := helpers.BuildBatcherFromArguments(c)
 	ext := helpers.BuildExtractorFromArguments(c, batcher)
@@ -130,6 +132,7 @@ func histogramCommand() *cli.Command {
 			helpers.SnapshotFlag,
 			helpers.NoOutFlag,
 			helpers.CSVFlag,
+			helpers.ScaleFlag,
 		},
 	})
 }

cmd/histo_test.go

@@ -10,6 +10,8 @@ func TestHistogram(t *testing.T) {
 	testCommandSet(t, histogramCommand(),
 		`-m (\d+) testdata/log.txt`,
 		`-m (\d+) testdata/graph.txt`,
+		`-m (\d+) --all testdata/graph.txt`,
+		`-m (\d+) --scale log10 testdata/graph.txt`,
 		`-o - -m (\d+) testdata/graph.txt`,
 		`-z -m (\d+) testdata/log.txt.gz`,
 	)

docs/usage/aggregators.md

@@ -49,6 +49,8 @@ of an extracted match.  That is to say, on every line a regex will be
 matched (or not), and the matched groups can be used to extract and build
 a key, that will act as the bucketing name.
 
+Supports [alternative scales](#alternative-scales)
+
 ### Example
 
 Extract HTTP verb, URL and status code.  Key off of status code and verb.
@@ -78,6 +80,8 @@ rare help bargraph
 Similar to histogram or table, bargraph can generate a stacked or grouped
 bargraph by one or two keys.
 
+Supports [alternative scales](#alternative-scales)
+
 ### Example
 
 !!! note "Color Coded Keys"

docs/usage/examples.md

@@ -50,12 +50,21 @@ Matched: 6 / 6 (Groups: 4)
 
 # Or with Bars/percentages
 ./rare histo --match "(\d+)" -e "{1}" -x simple.log
-1                   3          [50.0%] ||||||||||||||||||||||||||||||||||||||||||||||||||
-0                   1          [16.7%] ||||||||||||||||
-2                   1          [16.7%] ||||||||||||||||
-3                   1          [16.7%] ||||||||||||||||
+1                   7          [58.3%] ||||||||||||||||||||||||||||||||||||||||||||||||||
+3                   2          [16.7%] ||||||||||||||
+2                   2          [16.7%] ||||||||||||||
+0                   1          [ 8.3%] |||||||
 
-Matched: 6 / 6 (Groups: 4)
+Matched: 12 / 12 (Groups: 4)
+
+# Logarithmic Scale
+./rare histo --match "(\d+)" -e "{1}" --scale log10 -x simple.log
+1                   7          [58.3%] ||||||||||||||||||||||||||||||||||||||||||
+3                   2          [16.7%] |||||||||||||||
+2                   2          [16.7%] |||||||||||||||
+0                   1          [ 8.3%] 
+
+Matched: 12 / 12 (Groups: 4)
 ```
 
 ## Nginx
@@ -106,7 +115,7 @@ This shows an example of how to bucket the values into size of `1000`. In this c
 sense to see the histogram by number of bytes, but we might want to know the ratio of various orders-of-magnitudes.
 
 ```sh
-$ rare histo -m '"(\w{3,4}) ([A-Za-z0-9/.]+).*" (\d{3}) (\d+)' -e "{bucket {4} 10000}" -n 10 access.log -b
+$ rare histo -m '"(\w{3,4}) ([A-Za-z0-9/.]+).*" (\d{3}) (\d+)' -e "{bucket {4} 10000}" -n 10 -b access.log
 0                   144239     ||||||||||||||||||||||||||||||||||||||||||||||||||
 190000              2599       
 10000               1290       

docs/usage/expressions.md

@@ -356,12 +356,14 @@ Colorizes the 2nd argument.
 
 #### Bars
 
-Syntax: `{bar {val} "maxVal" "length"}`
+Syntax: `{bar {val} "maxVal" "length" ["scale"]}`
 
 Note: If unicode is disabled, will use pipe character
 
 Draws a "bar" with the length `(val / maxVal) * length`
 
+Scale can be `linear`, `log10`, or `log2` (Default: `lienar`)
+
 ### Paths
 
 Syntax: `{basename a/b/c}`, `{dirname a/b/c}`, `{extname a/b/c.jpg}`

pkg/expressions/stdlib/drawing.go

@@ -3,6 +3,7 @@ package stdlib
 import (
 	"rare/pkg/color"
 	. "rare/pkg/expressions" //lint:ignore ST1001 Legacy
+	"rare/pkg/multiterm/termscaler"
 	"rare/pkg/multiterm/termunicode"
 	"strconv"
 	"strings"
@@ -48,26 +49,41 @@ func kfRepeat(args []KeyBuilderStage) (KeyBuilderStage, error) {
 	}), nil
 }
 
-// {bar {val} "maxVal" "len"}
+// {bar {val} "maxVal" "len" ["scaler"]}
 func kfBar(args []KeyBuilderStage) (KeyBuilderStage, error) {
-	if len(args) != 3 {
-		return stageErrArgCount(args, 3)
+	if !isArgCountBetween(args, 3, 4) {
+		return stageErrArgRange(args, "3-4")
 	}
 
 	maxVal, maxValOk := EvalStageInt64(args[1])
 	if !maxValOk {
 		return stageArgError(ErrNum, 1)
 	}
-	maxLen, maxLenOk := EvalStageInt64(args[2])
+	maxLen, maxLenOk := EvalStageInt(args[2])
 	if !maxLenOk {
 		return stageArgError(ErrNum, 2)
 	}
 
+	scaler := termscaler.ScalerLinear
+	if len(args) >= 4 {
+		if name, ok := EvalStaticStage(args[3]); ok {
+			var scalerOk bool
+			if scaler, scalerOk = termscaler.ScalerByName(name); !scalerOk {
+				return stageArgError(ErrEnum, 3)
+			}
+		} else {
+			return stageArgError(ErrConst, 3)
+		}
+	}
+
 	return KeyBuilderStage(func(context KeyBuilderContext) string {
 		val, err := strconv.ParseInt(args[0](context), 10, 64)
 		if err != nil {
 			return ErrorNum
 		}
-		return termunicode.BarString(val, maxVal, maxLen)
+
+		var sb strings.Builder
+		termunicode.BarWrite(&sb, scaler.Scale(val, 0, maxVal), maxLen)
+		return sb.String()
 	}), nil
 }

pkg/expressions/stdlib/drawing_test.go

@@ -1,6 +1,8 @@
 package stdlib
 
 import (
+	"rare/pkg/multiterm/termunicode"
+	"rare/pkg/testutil"
 	"testing"
 )
 
@@ -29,10 +31,15 @@ func TestAddingColor(t *testing.T) {
 }
 
 func TestBarGraph(t *testing.T) {
-	testExpression(t,
-		mockContext(),
-		"{bar 2 5 5}",
-		"██")
+	testutil.SwitchGlobal(&termunicode.UnicodeEnabled, false)
+	defer testutil.RestoreGlobals()
+
+	testExpression(t, mockContext(), "{bar 2 5 5}", "||")
+	testExpression(t, mockContext(), "{bar 10 100 10}", "|")
+	testExpression(t, mockContext(), "{bar 10 100 10 log10}", "|||||")
+	testExpressionErr(t, mockContext(), "{bar 2 {0} 5}", "<BAD-TYPE>", ErrNum)
+	testExpressionErr(t, mockContext(), "{bar 10 100 10 badlog}", "<ENUM>", ErrEnum)
+	testExpressionErr(t, mockContext(), "{bar 10 100 10 {0}}", "<CONST>", ErrConst)
 }
 
 func TestBarGraphErrorCases(t *testing.T) {

pkg/multiterm/termrenderers/bargraph.go

@@ -5,6 +5,7 @@ import (
 	"rare/pkg/color"
 	"rare/pkg/humanize"
 	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termscaler"
 	"rare/pkg/multiterm/termunicode"
 	"strings"
 )
@@ -26,6 +27,7 @@ type BarGraph struct {
 
 	BarSize int
 	Stacked bool
+	Scaler  termscaler.Scaler
 }
 
 func NewBarGraph(term multiterm.MultilineTerm) *BarGraph {
@@ -34,6 +36,7 @@ func NewBarGraph(term multiterm.MultilineTerm) *BarGraph {
 		maxKeyLength: 4,
 		Stacked:      false,
 		BarSize:      50,
+		Scaler:       termscaler.ScalerLinear,
 	}
 }
 
@@ -143,7 +146,7 @@ func (s *BarGraph) writeBarGrouped(idx int, key string, vals ...int64) {
 			sb.WriteString(strings.Repeat(" ", s.maxKeyLength+2))
 		}
 		color.Write(&sb, color.GroupColors[i%len(color.GroupColors)], func(w io.StringWriter) {
-			termunicode.BarWrite(w, vals[i], s.maxLineVal, int64(s.BarSize))
+			termunicode.BarWrite(w, s.Scaler.Scale(vals[i], 0, s.maxLineVal), s.BarSize)
 		})
 		sb.WriteString(" ")
 		sb.WriteString(humanize.Hi(vals[i]))

pkg/multiterm/termrenderers/bargraph_test.go

@@ -2,6 +2,7 @@ package termrenderers
 
 import (
 	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termscaler"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -25,6 +26,25 @@ func TestBargraphRendering(t *testing.T) {
 	assert.Equal(t, "abc", v.Get(5))
 }
 
+func TestBargraphRenderingLog10(t *testing.T) {
+	v := multiterm.NewVirtualTerm()
+	bg := NewBarGraph(v)
+	bg.Stacked = false
+	bg.Scaler = termscaler.ScalerLog10
+
+	bg.SetKeys("a", "b")
+	bg.WriteBar(0, "test", 10, 100)
+	bg.WriteBar(1, "tes2", 50, 500)
+	bg.WriteFooter(0, "abc")
+
+	assert.Equal(t, "        0 a  1 b", v.Get(0))
+	assert.Equal(t, "test  ████████████████▊ 10", v.Get(1))
+	assert.Equal(t, "      █████████████████████████████████▍ 100", v.Get(2))
+	assert.Equal(t, "tes2  ████████████████████████████▎ 50", v.Get(3))
+	assert.Equal(t, "      █████████████████████████████████████████████ 500", v.Get(4))
+	assert.Equal(t, "abc", v.Get(5))
+}
+
 func TestBargraphStackedRendering(t *testing.T) {
 	v := multiterm.NewVirtualTerm()
 	bg := NewBarGraph(v)
@@ -59,7 +79,7 @@ func TestBargraphBadSubkeys(t *testing.T) {
 	assert.Equal(t, "abc", v.Get(4))
 }
 
-func TestBargraphUnicode(t *testing.T) {
+func TestBargraphUnicodeStacked(t *testing.T) {
 	v := multiterm.NewVirtualTerm()
 	bg := NewBarGraph(v)
 	bg.Stacked = true

pkg/multiterm/termrenderers/histoWriter.go

@@ -2,9 +2,11 @@ package termrenderers
 
 import (
 	"fmt"
+	"io"
 	"rare/pkg/color"
 	"rare/pkg/humanize"
 	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termscaler"
 	"rare/pkg/multiterm/termunicode"
 	"strings"
 )
@@ -23,13 +25,15 @@ type HistoWriter struct {
 
 	ShowBar        bool
 	ShowPercentage bool
+	Scaler         termscaler.Scaler
 }
 
 func NewHistogram(term multiterm.MultilineTerm, maxLines int) *HistoWriter {
 	return &HistoWriter{
 		writer:         term,
 		ShowBar:        true,
 		ShowPercentage: true,
+		Scaler:         termscaler.ScalerLinear,
 		textSpacing:    16,
 		items:          make([]histoPair, maxLines),
 	}
@@ -98,7 +102,9 @@ func (s *HistoWriter) writeLine(line int, key string, val int64) {
 
 	if s.ShowBar && s.maxVal > 0 {
 		sb.WriteString(" ")
-		sb.WriteString(color.Wrap(color.Blue, termunicode.BarString(val, s.maxVal, 50)))
+		color.Write(&sb, color.Blue, func(w io.StringWriter) {
+			termunicode.BarWrite(w, s.Scaler.Scale(val, 0, s.maxVal), 50)
+		})
 	}
 
 	s.writer.WriteForLine(line, sb.String())

pkg/multiterm/termscaler/scale.go

@@ -72,11 +72,21 @@ func Bucket(buckets int, unitVal float64) int {
 	return int(unitVal * float64(buckets-1))
 }
 
+// Return [0, maxLen]
+func LengthVal(maxLen int, unitVal float64) int {
+	return int(unitVal * float64(maxLen))
+}
+
 // Return [0, bucket-1]
 func (s Scaler) Bucket(buckets int, val, min, max int64) int {
 	return Bucket(buckets, s.Scale(val, min, max))
 }
 
+// Return [0, maxLen]
+func (s Scaler) LengthVal(maxLen int, val, min, max int64) int {
+	return LengthVal(maxLen, s.Scale(val, min, max))
+}
+
 // Returns scaled bucket values. May return less buckets than requested for small or high-curve ranges (won't return dupes)
 func (s Scaler) ScaleKeys(buckets, min, max int64) []int64 {
 	minf10, maxf10 := s.remapMinMax(min, max)