First pass at implementing log10/2 scale (#97)

Add log10/2 scale. Resolves #96

cmd/heatmap.go

@@ -15,15 +15,16 @@ import (
 
 func heatmapFunction(c *cli.Context) error {
 	var (
-		delim    = c.String("delim")
-		numRows  = c.Int("num")
-		numCols  = c.Int("cols")
-		minFixed = c.IsSet("min")
-		minVal   = c.Int64("min")
-		maxFixed = c.IsSet("max")
-		maxVal   = c.Int64("max")
-		sortRows = c.String("sort-rows")
-		sortCols = c.String("sort-cols")
+		delim      = c.String("delim")
+		numRows    = c.Int("num")
+		numCols    = c.Int("cols")
+		minFixed   = c.IsSet("min")
+		minVal     = c.Int64("min")
+		maxFixed   = c.IsSet("max")
+		maxVal     = c.Int64("max")
+		sortRows   = c.String("sort-rows")
+		sortCols   = c.String("sort-cols")
+		scalerName = c.String(helpers.ScaleFlag.Name)
 	)
 
 	counter := aggregation.NewTable(delim)
@@ -41,6 +42,7 @@ func heatmapFunction(c *cli.Context) error {
 	if minFixed || maxFixed {
 		writer.UpdateMinMax(minVal, maxVal)
 	}
+	writer.Scaler = helpers.BuildScalerOrFail(scalerName)
 
 	helpers.RunAggregationLoop(ext, counter, func() {
 		writer.WriteTable(counter, rowSorter, colSorter)
@@ -106,6 +108,7 @@ func heatmapCommand() *cli.Command {
 			helpers.SnapshotFlag,
 			helpers.NoOutFlag,
 			helpers.CSVFlag,
+			helpers.ScaleFlag,
 		},
 	})
 }

cmd/heatmap_test.go

@@ -1,10 +1,28 @@
 package cmd
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
 
 func TestHeatmap(t *testing.T) {
 	testCommandSet(t, heatmapCommand(),
 		`-m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`,
 		`-o - -m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`,
 	)
 }
+
+func TestHeatmapLinear(t *testing.T) {
+	out, eout, err := testCommandCapture(heatmapCommand(), `--snapshot -m "(.+) (.+)" -e "{$ {1} {2}}" testdata/heat.txt`)
+	assert.NoError(t, err)
+	assert.Empty(t, eout)
+	assert.Equal(t, " - 0    2 1    4 2    6 3    9 4\n a..\nx -22\ny 224\nz 9--\nMatched: 10 / 10 (R: 3; C: 3)\n39 B (0 B/s) \n", out)
+}
+
+func TestHeatmapLog2(t *testing.T) {
+	out, eout, err := testCommandCapture(heatmapCommand(), `--snapshot -m "(.+) (.+)" -e "{$ {1} {2}}" --scale log2 testdata/heat.txt`)
+	assert.NoError(t, err)
+	assert.Empty(t, eout)
+	assert.Equal(t, " - 1    4 2    7 3    9 4\n a..\nx ---\ny --4\nz 9--\nMatched: 10 / 10 (R: 3; C: 3)\n39 B (0 B/s) \n", out)
+}

cmd/helpers/extractorBuilder.go

@@ -17,6 +17,7 @@ const DefaultArgumentDescriptor = "<-|filename|glob...>"
 
 const (
 	cliCategoryRead     = "Input"
+	cliCategoryOutput   = "Output"
 	cliCategoryMatching = "Matching"
 	cliCategoryTweaking = "Tweaking"
 )

cmd/helpers/output.go

@@ -9,19 +9,22 @@ import (
 )
 
 var SnapshotFlag = &cli.BoolFlag{
-	Name:  "snapshot",
-	Usage: "In aggregators that support it, only output final results, and not progressive updates. Will enable automatically when piping output",
+	Name:     "snapshot",
+	Usage:    "In aggregators that support it, only output final results, and not progressive updates. Will enable automatically when piping output",
+	Category: cliCategoryOutput,
 }
 
 var CSVFlag = &cli.StringFlag{
-	Name:    "csv",
-	Aliases: []string{"o"},
-	Usage:   "Write final results to csv. Use - to output to stdout",
+	Name:     "csv",
+	Aliases:  []string{"o"},
+	Usage:    "Write final results to csv. Use - to output to stdout",
+	Category: cliCategoryOutput,
 }
 
 var NoOutFlag = &cli.BoolFlag{
-	Name:  "noout",
-	Usage: "Don't output any aggregation to stdout",
+	Name:     "noout",
+	Usage:    "Don't output any aggregation to stdout",
+	Category: cliCategoryOutput,
 }
 
 func BuildVTerm(forceSnapshot bool) multiterm.MultilineTerm {

cmd/helpers/scaler.go

@@ -0,0 +1,33 @@
+package helpers
+
+import (
+	"errors"
+	"rare/pkg/logger"
+	"rare/pkg/multiterm/termscaler"
+
+	"github.com/urfave/cli/v2"
+)
+
+var ScaleFlag = &cli.StringFlag{
+	Name:  "scale",
+	Usage: "Defines data-scaling (linear, log10, log2)",
+	Value: "linear",
+}
+
+func BuildScaler(scalerName string) (termscaler.Scaler, error) {
+	if scalerName == "" {
+		return termscaler.ScalerLinear, nil
+	}
+	if scaler, ok := termscaler.ScalerByName(scalerName); ok {
+		return scaler, nil
+	}
+	return termscaler.ScalerNull, errors.New("invalid scaler")
+}
+
+func BuildScalerOrFail(scalerName string) termscaler.Scaler {
+	s, err := BuildScaler(scalerName)
+	if err != nil {
+		logger.Fatal(err)
+	}
+	return s
+}

cmd/helpers/scaler_test.go

@@ -0,0 +1,23 @@
+package helpers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildScaler(t *testing.T) {
+	_, err := BuildScaler("")
+	assert.NoError(t, err)
+
+	_, err = BuildScaler("log10")
+	assert.NoError(t, err)
+
+	_, err = BuildScaler("bad-data")
+	assert.Error(t, err)
+}
+
+func TestBuildScalerOrFail(t *testing.T) {
+	assert.NotNil(t, BuildScalerOrFail(""))
+	assert.NotNil(t, BuildScalerOrFail("linear"))
+}

cmd/testdata/heat.txt

@@ -0,0 +1,10 @@
+a y
+a z
+a z
+a z
+a z
+b y
+b x
+c y
+c x
+c y

docs/usage/aggregators.md

@@ -190,6 +190,8 @@ Create a dense, color-coded, version of table-data by using cells to display
 the strength of a value. Can either use `\x00` or the `{$ a b}` helper.  First
 element is the column name, followed by the row name.
 
+Supports [alternative scales](#alternative-scales)
+
 ### Example
 
 ```bash
@@ -254,7 +256,9 @@ Matched: 1,035,666 / 1,035,666
 
 ---
 
-## Sorting
+## Common Arguments
+
+### Sorting
 
 Many of the aggregators support changing the order in which the data is displayed in. You
 can change this from default either by setting the `--sort` flag or `--sort-rows` and `--sort-cols`
@@ -268,7 +272,7 @@ These are the supported sorters:
 * `date` -- Parses the value as if it were a date. Falls back to contextual
 * `value` -- Orders the results based on their aggregated *value*. eg. would put the most frequent item at the top. Defaults to descending order
 
-### Modifiers
+#### Modifiers
 
 In addition to the sorting method, you can also modify the sort by adding a colon and the modifier, eg: `numeric:desc`
 
@@ -277,3 +281,9 @@ These are the supported modifiers:
 * `:reverse` -- Reverse of the "default"
 * `:asc` -- Ascending order
 * `:desc` -- Descending order
+
+### Alternative Scales
+
+Some of the aggregators support alternative display scaling: `linear` (default), `log10`, and `log2`.
+
+In aggregators that support it, you can specify with `--scale log10`

pkg/multiterm/termrenderers/heatmap.go

@@ -6,6 +6,7 @@ import (
 	"rare/pkg/color"
 	"rare/pkg/humanize"
 	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termscaler"
 	"rare/pkg/multiterm/termunicode"
 	"strings"
 )
@@ -17,6 +18,7 @@ type Heatmap struct {
 	FixedMin, FixedMax bool
 	maxRowKeyWidth     int // Max row width
 	currentRows        int // Currently used row count for non-footer
+	Scaler             termscaler.Scaler
 }
 
 func NewHeatmap(term multiterm.MultilineTerm, rows, cols int) *Heatmap {
@@ -26,6 +28,7 @@ func NewHeatmap(term multiterm.MultilineTerm, rows, cols int) *Heatmap {
 		term:           term,
 		maxRowKeyWidth: 0,
 		maxVal:         1,
+		Scaler:         termscaler.ScalerLinear,
 	}
 }
 
@@ -78,23 +81,14 @@ func (s *Heatmap) UpdateMinMax(min, max int64) {
 		sb.WriteRune(' ')
 	}
 
-	// Min
-	termunicode.HeatWriteLinear(&sb, s.minVal, s.minVal, s.maxVal)
-	sb.WriteString(" ")
-	sb.WriteString(humanize.Hi(s.minVal))
-
-	// mid-val
-	sb.WriteString("    ")
-	mid := s.minVal + (s.maxVal-s.minVal)/2
-	termunicode.HeatWriteLinear(&sb, mid, s.minVal, s.maxVal)
-	sb.WriteString(" ")
-	sb.WriteString(humanize.Hi(mid))
-
-	// Max
-	sb.WriteString("    ")
-	termunicode.HeatWriteLinear(&sb, s.maxVal, s.minVal, s.maxVal)
-	sb.WriteString(" ")
-	sb.WriteString(humanize.Hi(s.maxVal))
+	for idx, item := range s.Scaler.ScaleKeys(6, s.minVal, s.maxVal) {
+		if idx > 0 {
+			sb.WriteString("    ")
+		}
+		termunicode.HeatWrite(&sb, s.Scaler.Scale(item, s.minVal, s.maxVal))
+		sb.WriteString(" ")
+		sb.WriteString(humanize.Hi(item))
+	}
 
 	s.term.WriteForLine(0, sb.String())
 }
@@ -157,7 +151,7 @@ func (s *Heatmap) WriteRow(idx int, row *aggregation.TableRow, cols []string) {
 
 	for i := 0; i < len(cols); i++ {
 		val := row.Value(cols[i])
-		termunicode.HeatWriteLinear(&sb, val, s.minVal, s.maxVal)
+		termunicode.HeatWrite(&sb, s.Scaler.Scale(val, s.minVal, s.maxVal))
 	}
 
 	s.term.WriteForLine(2+idx, sb.String())

pkg/multiterm/termrenderers/heatmap_test.go

@@ -4,6 +4,7 @@ import (
 	"rare/pkg/aggregation"
 	"rare/pkg/aggregation/sorting"
 	"rare/pkg/multiterm"
+	"rare/pkg/multiterm/termscaler"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -20,7 +21,7 @@ func TestSimpleHeatmap(t *testing.T) {
 	hm.WriteTable(agg, sorting.NVNameSorter, sorting.NVNameSorter)
 
 	assert.Equal(t, 3, vt.LineCount())
-	assert.Equal(t, "     - 1    - 1    - 1", vt.Get(0))
+	assert.Equal(t, "     - 1    9 2", vt.Get(0))
 	assert.Equal(t, "     test", vt.Get(1))
 	assert.Equal(t, "abc  -", vt.Get(2))
 	assert.Equal(t, "", vt.Get(3))
@@ -39,12 +40,46 @@ func TestUnicodeHeatmap(t *testing.T) {
 	hm.WriteTable(agg, sorting.NVNameSorter, sorting.NVNameSorter)
 
 	assert.Equal(t, 4, vt.LineCount())
-	assert.Equal(t, "     - 0    - 0    9 1", vt.Get(0))
+	assert.Equal(t, "     - 0    9 1", vt.Get(0))
 	assert.Equal(t, "     a✤c", vt.Get(1))
 	assert.Equal(t, "test 99", vt.Get(2))
 	assert.Equal(t, "✤✥✦  9-", vt.Get(3))
 }
 
+func TestLogHeatmap(t *testing.T) {
+	vt := multiterm.NewVirtualTerm()
+	hm := NewHeatmap(vt, 4, 4)
+	hm.Scaler = termscaler.ScalerLog2
+
+	agg := aggregation.NewTable(" ")
+	agg.Sample("test abc")
+	agg.Sample("test1 abc")
+	agg.Sample("test1 abc")
+	agg.Sample("test1 abc")
+	agg.Sample("test1 abc")
+	agg.Sample("test2 abc")
+	agg.Sample("test32323 abc")
+	agg.Sample("test abc1")
+	agg.Sample("test abc1")
+	agg.Sample("test abc2")
+	agg.Sample("test abc3")
+	agg.Sample("test abc4")
+
+	hm.maxRowKeyWidth = 4
+	hm.WriteTable(agg, sorting.NVNameSorter, sorting.NVNameSorter)
+	hm.WriteFooter(0, "footer")
+
+	assert.Equal(t, 8, vt.LineCount())
+	assert.Equal(t, "     - 1    4 2    7 3    9 4", vt.Get(0))
+	assert.Equal(t, "     test", vt.Get(1))
+	assert.Equal(t, "abc  -9--", vt.Get(2))
+	assert.Equal(t, "abc1 4---", vt.Get(3))
+	assert.Equal(t, "abc2 ----", vt.Get(4))
+	assert.Equal(t, "abc3 ----", vt.Get(5))
+	assert.Equal(t, "(1 more)", vt.Get(6))
+	assert.Equal(t, "footer", vt.Get(7))
+}
+
 func TestCompressedHeatmap(t *testing.T) {
 	vt := multiterm.NewVirtualTerm()
 	hm := NewHeatmap(vt, 2, 2)
@@ -64,7 +99,7 @@ func TestCompressedHeatmap(t *testing.T) {
 	hm.WriteFooter(0, "footer")
 
 	assert.Equal(t, 6, vt.LineCount())
-	assert.Equal(t, "     - 0    - 0    9 1", vt.Get(0))
+	assert.Equal(t, "     - 0    9 1", vt.Get(0))
 	assert.Equal(t, "     test (2 more)", vt.Get(1))
 	assert.Equal(t, "abc  99", vt.Get(2))
 	assert.Equal(t, "abc1 9-", vt.Get(3))