📂 add the ability to select/clear unused cache items

README.md

@@ -314,6 +314,12 @@ snowman cache list-of-icecream.rq --invalidate
 snowman cache icecream.rq "your parameter" --invalidate
 ```
 
+Sometimes following changes to your queries and external data, you can end up having unused cache items. You can clear these using the `--unused` selector flag.
+
+```bash
+snowman cache --unused --invalidate
+```
+
 ### Using the built-in server
 
 Snowman comes with a built-in development server exposed through the `server` command. The `server` command has two optional arguments `port` and `address` which one can use to bind Snowman to specified IP addresses and ports.

cmd/build.go

@@ -145,6 +145,10 @@ var buildCmd = &cobra.Command{
 
 		}
 
+		if err := sparql.CurrentRepository.CacheManager.Teardown(); err != nil {
+			return utils.ErrorExit("Failed write used queries to cache memory.", err)
+		}
+
 		return nil
 	},
 }

cmd/cache.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/glaciers-in-archives/snowman/internal/cache"
@@ -12,6 +13,7 @@ import (
 )
 
 var invalidateCacheOption bool
+var unusedOption bool
 
 func printFileContents(path string) error {
 	fmt.Println(path)
@@ -43,26 +45,45 @@ var cacheCmd = &cobra.Command{
 	Long:  `This command allows you to inspect the cache for any cached query. The first argument should be the name of the SPARQL query. To inspect the cache of a parameterized query provide a second argument with its parameter value.`,
 	Args:  cobra.RangeArgs(0, 2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if len(args) == 0 {
-			if invalidateCacheOption {
-				err := os.RemoveAll(cache.CacheLocation)
-				if err != nil {
-					return utils.ErrorExit("Failed to remove directory.", err)
-				}
-				return nil
+		var selectedCacheItems []string
+
+		if len(args) == 0 && !unusedOption {
+			totFiles, err := utils.CountFilesRecursive(cache.CacheLocation)
+			if err != nil {
+				return utils.ErrorExit("Failed to retrive cache info.", err)
 			}
 
-			fmt.Println("No arguments or flags given.")
-		} else if len(args) == 1 {
-			dirPath := cache.CacheLocation + cache.Hash(args[0])
+			fmt.Println("There are " + fmt.Sprint(totFiles) + " cache items.")
 
-			if invalidateCacheOption {
-				err := os.RemoveAll(dirPath)
+			selectedCacheItems = append(selectedCacheItems, cache.CacheLocation)
+		} else if len(args) == 0 && unusedOption {
+			usedItems, err := utils.ReadLineSeperatedFile(".snowman/last_build_queries.txt")
+			if err != nil {
+				return utils.ErrorExit("Failed to read last unused cache items: ", err)
+			}
+
+			err = filepath.Walk(cache.CacheLocation, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
-					return utils.ErrorExit("Failed to remove directory.", err)
+					return err
+				}
+
+				pathAsCacheItem := strings.Replace(strings.Replace(path, ".json", "", 1), ".snowman/cache/", "", 1)
+				isUsed := false
+				for _, used := range usedItems {
+					if pathAsCacheItem == used || strings.HasPrefix(used, pathAsCacheItem) {
+						isUsed = true
+					}
+				}
+
+				if !isUsed {
+					selectedCacheItems = append(selectedCacheItems, path)
 				}
 				return nil
-			}
+			})
+
+			fmt.Println("Found " + fmt.Sprint(len(selectedCacheItems)) + " unused cache items.")
+		} else if len(args) == 1 {
+			dirPath := cache.CacheLocation + cache.Hash(args[0])
 
 			files, err := os.ReadDir(dirPath)
 			if err != nil {
@@ -71,10 +92,11 @@ var cacheCmd = &cobra.Command{
 
 			if len(files) > 1 {
 				fmt.Println(args[0] + " represents a parameterized query with " + fmt.Sprint(len(files)) + " cache items.")
-				return nil
+			} else {
+				printFileContents(dirPath + "/" + files[0].Name())
 			}
 
-			return printFileContents(dirPath + "/" + files[0].Name())
+			selectedCacheItems = append(selectedCacheItems, dirPath)
 		} else if len(args) == 2 {
 
 			sparqlBytes, err := ioutil.ReadFile("queries/" + args[0])
@@ -85,15 +107,18 @@ var cacheCmd = &cobra.Command{
 			queryString := strings.Replace(string(sparqlBytes), "{{.}}", args[1], 1)
 
 			filePath := cache.CacheLocation + cache.Hash(args[0]) + "/" + cache.Hash(queryString) + ".json"
-			if invalidateCacheOption {
-				if err := os.Remove(filePath); err != nil {
+			selectedCacheItems = append(selectedCacheItems, filePath)
+
+			printFileContents((filePath))
+		}
+
+		if invalidateCacheOption {
+			for _, item := range selectedCacheItems {
+				if err := os.RemoveAll(item); err != nil {
+					fmt.Println("Removing: " + item)
 					return utils.ErrorExit("Failed to remove the cache file.", err)
 				}
-
-				return nil
 			}
-
-			return printFileContents((filePath))
 		}
 
 		return nil
@@ -103,4 +128,5 @@ var cacheCmd = &cobra.Command{
 func init() {
 	rootCmd.AddCommand(cacheCmd)
 	cacheCmd.Flags().BoolVarP(&invalidateCacheOption, "invalidate", "i", false, "Removes/clears the specified parts of the query cache.")
+	cacheCmd.Flags().BoolVarP(&unusedOption, "unused", "u", false, "Returns cache items not used in the last build.")
 }

internal/cache/cache.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/glaciers-in-archives/snowman/internal/utils"
 )
 
 var CacheLocation string = ".snowman/cache/"
@@ -17,46 +19,57 @@ func Hash(value string) string {
 }
 
 type CacheManager struct {
-	CacheStrategy string // "available", "never"
-	CacheHashes   map[string]bool
+	CacheStrategy          string // "available", "never"
+	StoredCacheHashes      map[string]bool
+	CacheHashesUsedInBuild map[string]bool
 }
 
 func NewCacheManager(strategy string) (*CacheManager, error) {
 	cm := CacheManager{
 		CacheStrategy: strategy,
 	}
-	cm.CacheHashes = make(map[string]bool)
+	cm.StoredCacheHashes = make(map[string]bool)
+	cm.CacheHashesUsedInBuild = make(map[string]bool)
 
 	if err := os.MkdirAll(CacheLocation, 0770); err != nil {
 		return nil, err
 	}
 
 	if strategy != "never" {
-		// index cache hashes
-		locationHashes, err := ioutil.ReadDir(CacheLocation)
-		if err != nil {
+		if err := cm.readStoredHashes(); err != nil {
 			return nil, err
 		}
+	}
+
+	return &cm, nil
+}
+
+func (cm *CacheManager) readStoredHashes() error {
+	// index cache hashes
+	locationHashes, err := ioutil.ReadDir(CacheLocation)
+	if err != nil {
+		return err
+	}
 
-		for _, locationDirInfo := range locationHashes {
-			contentDirInfo, err := ioutil.ReadDir(CacheLocation + locationDirInfo.Name())
-			if err != nil {
-				return nil, err
-			}
-			for _, contentFileInfo := range contentDirInfo {
-				fullCacheHash := locationDirInfo.Name() + "/" + strings.Replace(contentFileInfo.Name(), ".json", "", 1)
-				cm.CacheHashes[fullCacheHash] = true
-			}
+	for _, locationDirInfo := range locationHashes {
+		contentDirInfo, err := ioutil.ReadDir(CacheLocation + locationDirInfo.Name())
+		if err != nil {
+			return err
+		}
+		for _, contentFileInfo := range contentDirInfo {
+			fullCacheHash := locationDirInfo.Name() + "/" + strings.Replace(contentFileInfo.Name(), ".json", "", 1)
+			cm.StoredCacheHashes[fullCacheHash] = true
 		}
 	}
 
-	return &cm, nil
+	return nil
 }
 
 func (cm *CacheManager) GetCache(location string, query string) (*os.File, error) {
 	fullQueryHash := Hash(location) + "/" + Hash(query)
+	cm.CacheHashesUsedInBuild[fullQueryHash] = true
 
-	if !cm.CacheHashes[fullQueryHash] || cm.CacheStrategy == "never" {
+	if !cm.StoredCacheHashes[fullQueryHash] || cm.CacheStrategy == "never" {
 		return nil, nil
 	}
 
@@ -90,7 +103,15 @@ func (cm *CacheManager) SetCache(location string, query string, content string)
 	defer f.Close()
 	f.Sync()
 
-	cm.CacheHashes[fullQueryHash] = true
+	cm.StoredCacheHashes[fullQueryHash] = true
+
+	return nil
+}
+
+func (cm *CacheManager) Teardown() error {
+	if err := utils.WriteLineSeperatedFile(cm.CacheHashesUsedInBuild, ".snowman/last_build_queries.txt"); err != nil {
+		return err
+	}
 
 	return nil
 }

internal/utils/utils.go

@@ -1,8 +1,10 @@
 package utils
 
 import (
+	"bufio"
 	"errors"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -52,3 +54,61 @@ func CopyFile(srcFile, dstFile string) error {
 
 	return nil
 }
+
+func WriteLineSeperatedFile(data map[string]bool, path string) error {
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+
+	writer := bufio.NewWriter(file)
+
+	for value := range data {
+		_, err := writer.WriteString(value + "\n")
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := writer.Flush(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ReadLineSeperatedFile(path string) ([]string, error) {
+	bytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	values := strings.Split(string(bytes), "\n")
+	return values, nil
+}
+
+func CountFilesRecursive(dir string) (int, error) {
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		return 0, nil
+	}
+
+	count := 0
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			count += 1
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return 0, err
+	}
+
+	return count, nil
+}