adding cacheTimeout, and refactoring index to only return enriched da…

…ta that is needed

cmd/root.go

@@ -41,6 +41,7 @@ import (
 
 var (
 	//cfgFile       string
+	cacheTimeout  time.Duration
 	clientTimeout time.Duration
 	cliTimeout    time.Duration
 	columns       string
@@ -64,6 +65,7 @@ var (
 
 func init() {
 	//cobra.OnInitialize(initConfig)
+	rootCmd.PersistentFlags().DurationVarP(&cacheTimeout, "cache_timeout", "", time.Minute*5, "timeout duration for local db (db.bolt) cache file")
 	rootCmd.PersistentFlags().StringVarP(&foo, "foo", "", "blah", "foo is a test flag")
 	rootCmd.PersistentFlags().StringVarP(&slice, "slice", "", "", "slices the results after filtering followed by sorting")
 	rootCmd.PersistentFlags().StringVarP(&sortOrder, "sort", "s", "",
@@ -222,6 +224,7 @@ func packageCfg(args []string) *pkg.ConfigCtx {
 	}
 
 	cfgCtx := pkg.NewConfigCtx()
+	cfgCtx.CacheTimeout = cacheTimeout
 	cfgCtx.IPsOutput = ips
 	cfgCtx.IPsDelimiter = ips_delimiter
 	cfgCtx.JsonOutput = jsonn

pkg/config_ctx.go

@@ -104,6 +104,7 @@ type TailscaleCLICfgCtx struct {
 }
 
 type ConfigCtx struct {
+	CacheTimeout  time.Duration
 	Columns       mapset.Set[string]
 	Concurrency   int
 	Filters       map[string]mapset.Set[string]

pkg/db.go

@@ -30,6 +30,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 	"tips/pkg/tailscale_cli"
 
@@ -48,8 +49,16 @@ const (
 	// These two buckets contain FULL data.
 	devicesBucket  = "devices.full"
 	enrichedBucket = "enriched.full"
+
+	statsBucket = "stats"
+	statsKey    = "stats"
 )
 
+type DBStats struct {
+	DevicesCount  int `json:"devices_count"`
+	EnrichedCount int `json:"enriched_count"`
+}
+
 type DB struct {
 	tailnetScope string
 	hdl          *bolt.DB
@@ -91,13 +100,18 @@ func (d *DB) Erase() error {
 }
 
 func (d *DB) Exists(ctx context.Context) (bool, error) {
-	return fileExistsAndIsRecent(d.File(), time.Second*15)
+	cfg := CtxAsConfig(ctx, CtxKeyConfig)
+	return fileExistsAndIsRecent(d.File(), cfg.CacheTimeout)
 }
 
 func (d *DB) IndexDevices(ctx context.Context, devList []tailscale.Device, enrichedDevList map[string]tailscale_cli.DeviceInfo) error {
 	// Start a writable transaction.
 	err := d.hdl.Update(func(tx *bolt.Tx) error {
 		// Create all buckets.
+		statsBuck, err := tx.CreateBucketIfNotExists([]byte(statsBucket))
+		if err != nil {
+			return err
+		}
 
 		// If the bucket already exists, it will return a reference to it.
 		devicesBucket, err := tx.CreateBucketIfNotExists([]byte(devicesBucket))
@@ -110,6 +124,22 @@ func (d *DB) IndexDevices(ctx context.Context, devList []tailscale.Device, enric
 			return err
 		}
 
+		// Record stats
+		// TODO: record version
+		var stats DBStats
+		stats.DevicesCount = len(devList)
+		stats.EnrichedCount = len(enrichedDevList)
+
+		encoded, err := json.Marshal(stats)
+		if err != nil {
+			return err
+		}
+
+		err = statsBuck.Put([]byte(statsKey), encoded)
+		if err != nil {
+			return err
+		}
+
 		// Iterate over all devices.
 		for _, dev := range devList {
 			// Encode as JSON: in the future encode as Proto/more compact form.
@@ -155,9 +185,26 @@ func (d *DB) IndexDevices(ctx context.Context, devList []tailscale.Device, enric
 func (d *DB) FindDevices(ctx context.Context) ([]tailscale.Device, map[string]tailscale_cli.DeviceInfo, error) {
 	// 0. First populate all devices.
 	// TODO: index metadata like the size of devices then we can instantiate with the correct capacity.
-	devList := make([]tailscale.Device, 0)
+	//devList := make([]tailscale.Device, 0)
+	var devList []tailscale.Device
+	var enrichedDevs map[string]tailscale_cli.DeviceInfo
 	err := d.hdl.View(func(tx *bolt.Tx) error {
-		// Assume bucket exists and has keys
+		// 0. Check for stats
+		sb := tx.Bucket([]byte(statsBucket))
+		if sb == nil {
+			return errors.New("bucket is unknown: " + statsBucket)
+		}
+		sts := sb.Get([]byte(statsKey))
+		var stats DBStats
+		err := json.Unmarshal(sts, &stats)
+		if err != nil {
+			return err
+		}
+
+		devList = make([]tailscale.Device, 0, stats.DevicesCount)
+		enrichedDevs = make(map[string]tailscale_cli.DeviceInfo, stats.EnrichedCount)
+
+		// 1. Next populate all devices data
 		b := tx.Bucket([]byte(devicesBucket))
 		if b == nil {
 			return errors.New("bucket is unknown: " + devicesBucket)
@@ -178,38 +225,20 @@ func (d *DB) FindDevices(ctx context.Context) ([]tailscale.Device, map[string]ta
 			devList = append(devList, dev)
 		}
 
-		// TODO: need to set this up.
-		// Prefix scan (use this in the future)
-		//prefix := []byte("1234")
-		//for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
-		//	fmt.Printf("key=%s, value=%s\n", k, v)
-		//}
-
-		return nil
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// 2. Next populate all enriched data
-	enrichedDevs := make(map[string]tailscale_cli.DeviceInfo)
-	err = d.hdl.View(func(tx *bolt.Tx) error {
-		// Assume bucket exists and has keys
-		b := tx.Bucket([]byte(enrichedBucket))
-		c := b.Cursor()
-
-		// This is a linear scan over all key/values
-		for k, v := c.First(); k != nil; k, v = c.Next() {
-			//fmt.Printf("key=%s, value=%s\n", k, v)
-			//fmt.Printf("key=%s\n", k)
+		// 2. Next populate only enriched data, that is needed.
+		// No cursor is needed, we only need to get the enriched data for devices that were returned above!
+		b = tx.Bucket([]byte(enrichedBucket))
 
+		for _, dev := range devList {
+			k := dev.NodeKey
+			v := b.Get([]byte(k))
 			var dev tailscale_cli.DeviceInfo
 			err := json.Unmarshal(v, &dev)
 			if err != nil {
 				return err
 			}
 
-			enrichedDevs[string(k)] = dev
+			enrichedDevs[k] = dev
 		}
 
 		// TODO: need to set this up.
@@ -222,8 +251,30 @@ func (d *DB) FindDevices(ctx context.Context) ([]tailscale.Device, map[string]ta
 		return nil
 	})
 	if err != nil {
-		log.Fatal("error occurred attempting to lookup enriched devices", "error", err)
+		return nil, nil, err
 	}
 
 	return devList, enrichedDevs, nil
 }
+
+func fileExistsAndIsRecent(filePath string, duration time.Duration) (bool, error) {
+	// Check if the file exists
+	info, err := os.Stat(filePath)
+	if os.IsNotExist(err) {
+		// The file does not exist
+		return false, nil
+	} else if err != nil {
+		// There was some other error getting the file info
+		return false, err
+	}
+
+	// Check the time since the file was created
+	creationTime := info.ModTime()
+	if time.Since(creationTime) <= duration {
+		// The file is recent enough
+		return true, nil
+	}
+
+	// The file exists but is not recent
+	return false, nil
+}

pkg/resources.go

@@ -27,36 +27,13 @@ package pkg
 
 import (
 	"context"
-	"os"
 	"time"
 	"tips/pkg/tailscale_cli"
 
 	"github.com/charmbracelet/log"
 	"github.com/tailscale/tailscale-client-go/tailscale"
 )
 
-func fileExistsAndIsRecent(filePath string, duration time.Duration) (bool, error) {
-	// Check if the file exists
-	info, err := os.Stat(filePath)
-	if os.IsNotExist(err) {
-		// The file does not exist
-		return false, nil
-	} else if err != nil {
-		// There was some other error getting the file info
-		return false, err
-	}
-
-	// Check the time since the file was created
-	creationTime := info.ModTime()
-	if time.Since(creationTime) <= duration {
-		// The file is recent enough
-		return true, nil
-	}
-
-	// The file exists but is not recent
-	return false, nil
-}
-
 func DevicesResource(ctx context.Context, client *tailscale.Client) ([]tailscale.Device, map[string]tailscale_cli.DeviceInfo, error) {
 	cfg := CtxAsConfig(ctx, CtxKeyConfig)
 	startTime := time.Now()
@@ -65,8 +42,7 @@ func DevicesResource(ctx context.Context, client *tailscale.Client) ([]tailscale
 	}()
 
 	// 0. Check this index first.
-	indexedDB := NewDB("deckarep@gmail.com")
-
+	indexedDB := NewDB(cfg.Tailnet)
 	existsAndRecent, err := indexedDB.Exists(ctx)
 	if err != nil {
 		log.Warn("problem checking for bolt db file", "error", err)
@@ -80,12 +56,12 @@ func DevicesResource(ctx context.Context, client *tailscale.Client) ([]tailscale
 
 	// TODO: 0. Check cache config - return cached results if cache timeout not yet expired.
 	if cfg.NoCache {
-		log.Info("--nocache was supplied, so forcing a fetch of all data")
+		log.Info("--nocache was supplied, so forcing a fresh fetch of all data")
 	} else if devList, enrichedDevs, err := indexedDB.FindDevices(ctx); existsAndRecent && err == nil {
-		log.Info("found a bolt file and its recent enough so that will be used!")
+		log.Info("local db file (db.bolt) was found and recent enough so using this as a cache")
 		return devList, enrichedDevs, nil
 	} else {
-		log.Info("bolt file has expired or must be regenerated")
+		log.Info("local db file (db.bolt) has expired or must be regenerated")
 	}
 
 	// 1. Do tailscale api lookup for devices data.