HardlinkedOutsideClient torrent field

cmd/clean.go

@@ -118,7 +118,7 @@ var cleanCmd = &cobra.Command{
 		tfm := torrentfilemap.New(torrents)
 		log.Infof("Mapped torrents to %d unique torrent files", tfm.Length())
 
-		// create map of paths associated to underlying file ids
+		// download path mapping
 		clientDownloadPathMapping, err := getClientDownloadPathMapping(clientConfig)
 		if err != nil {
 			log.WithError(err).Fatal("Failed loading client download path mappings")
@@ -127,10 +127,17 @@ var cleanCmd = &cobra.Command{
 				clientDownloadPathMapping)
 		}
 
+		// create map of paths associated to underlying file ids
 		start := time.Now()
 		hfm := hardlinkfilemap.New(torrents, clientDownloadPathMapping)
 		log.Infof("Mapped all torrent file paths to %d unique underlying file IDs in %s", hfm.Length(), time.Since(start))
 
+		// add HardlinkedOutsideClient field to torrents
+		for h, t := range torrents {
+			t.HardlinkedOutsideClient = hfm.HardlinkedOutsideClient(t)
+			torrents[h] = t
+		}
+
 		// remove torrents that are not ignored and match remove criteria
 		if err := removeEligibleTorrents(log, c, torrents, tfm, hfm); err != nil {
 			log.WithError(err).Fatal("Failed removing eligible torrents...")

cmd/relabel.go

@@ -2,13 +2,15 @@ package cmd
 
 import (
 	"encoding/json"
+	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/spf13/cobra"
 
 	"github.com/l3uddz/tqm/client"
 	"github.com/l3uddz/tqm/config"
 	"github.com/l3uddz/tqm/expression"
+	"github.com/l3uddz/tqm/hardlinkfilemap"
 	"github.com/l3uddz/tqm/logger"
 	"github.com/l3uddz/tqm/torrentfilemap"
 	"github.com/l3uddz/tqm/tracker"
@@ -116,6 +118,25 @@ var relabelCmd = &cobra.Command{
 		tfm := torrentfilemap.New(torrents)
 		log.Infof("Mapped torrents to %d unique torrent files", tfm.Length())
 
+		// download path mapping
+		clientDownloadPathMapping, err := getClientDownloadPathMapping(clientConfig)
+		if err != nil {
+			log.WithError(err).Fatal("Failed loading client download path mappings")
+		} else if clientDownloadPathMapping != nil {
+			log.Debugf("Loaded %d client download path mappings: %#v", len(clientDownloadPathMapping),
+				clientDownloadPathMapping)
+		}
+
+		// create map of paths associated to underlying file ids
+		start := time.Now()
+		hfm := hardlinkfilemap.New(torrents, clientDownloadPathMapping)
+		log.Infof("Mapped all torrent file paths to %d unique underlying file IDs in %s", hfm.Length(), time.Since(start))
+
+		// add HardlinkedOutsideClient field to torrents
+		for _, t := range torrents {
+			t.HardlinkedOutsideClient = hfm.HardlinkedOutsideClient(t)
+		}
+
 		// relabel torrents that meet the filter criteria
 		if err := relabelEligibleTorrents(log, c, torrents, tfm); err != nil {
 			log.WithError(err).Fatal("Failed relabeling eligible torrents...")

cmd/retag.go

@@ -2,13 +2,15 @@ package cmd
 
 import (
 	"encoding/json"
+	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/spf13/cobra"
 
 	"github.com/l3uddz/tqm/client"
 	"github.com/l3uddz/tqm/config"
 	"github.com/l3uddz/tqm/expression"
+	"github.com/l3uddz/tqm/hardlinkfilemap"
 	"github.com/l3uddz/tqm/logger"
 	"github.com/l3uddz/tqm/torrentfilemap"
 	"github.com/l3uddz/tqm/tracker"
@@ -126,6 +128,25 @@ var retagCmd = &cobra.Command{
 		tfm := torrentfilemap.New(torrents)
 		log.Infof("Mapped torrents to %d unique torrent files", tfm.Length())
 
+		// download path mapping
+		clientDownloadPathMapping, err := getClientDownloadPathMapping(clientConfig)
+		if err != nil {
+			log.WithError(err).Fatal("Failed loading client download path mappings")
+		} else if clientDownloadPathMapping != nil {
+			log.Debugf("Loaded %d client download path mappings: %#v", len(clientDownloadPathMapping),
+				clientDownloadPathMapping)
+		}
+
+		// create map of paths associated to underlying file ids
+		start := time.Now()
+		hfm := hardlinkfilemap.New(torrents, clientDownloadPathMapping)
+		log.Infof("Mapped all torrent file paths to %d unique underlying file IDs in %s", hfm.Length(), time.Since(start))
+
+		// add HardlinkedOutsideClient field to torrents
+		for _, t := range torrents {
+			t.HardlinkedOutsideClient = hfm.HardlinkedOutsideClient(t)
+		}
+
 		// Verify tags exist on client
 		var tagList []string = []string{}
 		for _, v := range exp.Tags {

config/torrent.go

@@ -50,6 +50,9 @@ type Torrent struct {
 	// tracker
 	TrackerName   string `json:"TrackerName"`
 	TrackerStatus string `json:"TrackerStatus"`
+
+	// set by command
+	HardlinkedOutsideClient bool `json:"-"`
 }
 
 func (t *Torrent) IsUnregistered() bool {

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/scylladb/go-set v1.0.2
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tcnksm/go-gitconfig v0.1.2 // indirect
 	golang.org/x/text v0.3.7 // indirect

go.sum

@@ -35,6 +35,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -173,6 +174,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
+github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
+github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

hardlinkfilemap/hardlinkfilemap.go

@@ -6,12 +6,12 @@ import (
 
 	"github.com/l3uddz/tqm/config"
 	"github.com/l3uddz/tqm/logger"
-	"github.com/l3uddz/tqm/sliceutils"
+	"github.com/scylladb/go-set/strset"
 )
 
 func New(torrents map[string]config.Torrent, torrentPathMapping map[string]string) *HardlinkFileMap {
 	tfm := &HardlinkFileMap{
-		hardlinkFileMap:    make(map[string][]string),
+		hardlinkFileMap:    make(map[string]*strset.Set),
 		log:                logger.GetLogger("hardlinkfilemap"),
 		torrentPathMapping: torrentPathMapping,
 	}
@@ -33,62 +33,59 @@ func (t *HardlinkFileMap) ConsiderPathMapping(path string) string {
 	return path
 }
 
-func (t *HardlinkFileMap) FileIdentifierByPath(path string) (string, bool) {
+func (t *HardlinkFileMap) LinkInfoByPath(path string) (string, uint64, bool) {
 	stat, err1 := os.Stat(path)
 	if err1 != nil {
 		t.log.Warnf("Failed to stat file: %s - %s", path, err1)
-		return "", false
+		return "", 0, false
 	}
 
-	id, err2 := FileIdentifier(stat)
+	id, nlink, err2 := LinkInfo(stat, path)
 	if err2 != nil {
 		t.log.Warnf("Failed to get file identifier: %s - %s", path, err2)
-		return "", false
+		return "", 0, false
 	}
 
-	return id, true
+	return id, nlink, true
 }
 
 func (t *HardlinkFileMap) AddByTorrent(torrent config.Torrent) {
 	for _, f := range torrent.Files {
 		f = t.ConsiderPathMapping(f)
 
-		id, ok := t.FileIdentifierByPath(f)
+		id, _, ok := t.LinkInfoByPath(f)
 
 		if !ok {
 			continue
 		}
 
 		if _, exists := t.hardlinkFileMap[id]; exists {
 			// file id already associated with other paths
-			t.hardlinkFileMap[id] = append(t.hardlinkFileMap[id], f)
+			t.hardlinkFileMap[id].Add(f)
 			continue
 		}
 
 		// file id has not been seen before, create id entry
-		t.hardlinkFileMap[id] = []string{f}
+		t.hardlinkFileMap[id] = strset.New(f)
 	}
 }
 
 func (t *HardlinkFileMap) RemoveByTorrent(torrent config.Torrent) {
 	for _, f := range torrent.Files {
 		f = t.ConsiderPathMapping(f)
 
-		id, ok := t.FileIdentifierByPath(f)
+		id, _, ok := t.LinkInfoByPath(f)
 
 		if !ok {
 			continue
 		}
 
 		if _, exists := t.hardlinkFileMap[id]; exists {
 			// remove this path from the id entry
-			i := sliceutils.IndexOfString(t.hardlinkFileMap[id], f)
-			if i != -1 {
-				t.hardlinkFileMap[id] = sliceutils.FastDelete(t.hardlinkFileMap[id], i)
-			}
+			t.hardlinkFileMap[id].Remove(f)
 
 			// remove id entry if no more paths
-			if len(t.hardlinkFileMap[id]) == 0 {
+			if t.hardlinkFileMap[id].Size() == 0 {
 				delete(t.hardlinkFileMap, id)
 			}
 
@@ -97,17 +94,44 @@ func (t *HardlinkFileMap) RemoveByTorrent(torrent config.Torrent) {
 	}
 }
 
-func (t *HardlinkFileMap) IsTorrentUnique(torrent config.Torrent) bool {
+func (t *HardlinkFileMap) CountLinks(f string) (inmap uint64, total uint64, ok bool) {
+	f = t.ConsiderPathMapping(f)
+	id, nlink, ok := t.LinkInfoByPath(f)
+
+	if !ok {
+		return 0, 0, false
+	}
+
+	if paths, exists := t.hardlinkFileMap[id]; exists {
+		return uint64(paths.Size()), nlink, true
+	}
+
+	return 0, nlink, true
+}
+
+func (t *HardlinkFileMap) HardlinkedOutsideClient(torrent config.Torrent) bool {
 	for _, f := range torrent.Files {
-		f = t.ConsiderPathMapping(f)
+		inmap, total, ok := t.CountLinks(f)
+		if !ok {
+			continue
+		}
+
+		if total != inmap {
+			return true
+		}
+	}
 
-		id, ok := t.FileIdentifierByPath(f)
+	return false
+}
 
+func (t *HardlinkFileMap) IsTorrentUnique(torrent config.Torrent) bool {
+	for _, f := range torrent.Files {
+		c, _, ok := t.CountLinks(f)
 		if !ok {
 			return false
 		}
 
-		if paths, exists := t.hardlinkFileMap[id]; exists && len(paths) > 1 {
+		if c > 1 {
 			return false
 		}
 	}
@@ -117,15 +141,12 @@ func (t *HardlinkFileMap) IsTorrentUnique(torrent config.Torrent) bool {
 
 func (t *HardlinkFileMap) NoInstances(torrent config.Torrent) bool {
 	for _, f := range torrent.Files {
-		f = t.ConsiderPathMapping(f)
-
-		id, ok := t.FileIdentifierByPath(f)
-
+		c, _, ok := t.CountLinks(f)
 		if !ok {
 			return false
 		}
 
-		if paths, exists := t.hardlinkFileMap[id]; exists && len(paths) != 0 {
+		if c != 0 {
 			return false
 		}
 	}

hardlinkfilemap/fileidentifier_darwin.go → hardlinkfilemap/linkinfo_darwin.go

@@ -7,11 +7,13 @@ import (
 	"syscall"
 )
 
-func FileIdentifier(fi os.FileInfo) (string, error) {
+func LinkInfo(fi os.FileInfo, _ string) (string, uint64, error) {
 	sys, ok := fi.Sys().(*syscall.Stat_t)
 	if !ok {
-		return "", errors.New("failed to get file identifier")
+		return "", 0, errors.New("failed to get file identifier")
 	}
 
-	return strconv.FormatInt(int64(sys.Dev), 10) + "|" + strconv.FormatUint(sys.Ino, 10), nil
+	return strconv.FormatInt(int64(sys.Dev), 10) + "|" + strconv.FormatUint(sys.Ino, 10),
+		uint64(sys.Nlink),
+		nil
 }

hardlinkfilemap/fileidentifier_linux.go → hardlinkfilemap/linkinfo_linux.go

@@ -7,11 +7,13 @@ import (
 	"syscall"
 )
 
-func FileIdentifier(fi os.FileInfo) (string, error) {
+func LinkInfo(fi os.FileInfo, _ string) (string, uint64, error) {
 	sys, ok := fi.Sys().(*syscall.Stat_t)
 	if !ok {
-		return "", errors.New("failed to get file identifier")
+		return "", 0, errors.New("failed to get file identifier")
 	}
 
-	return strconv.FormatUint(sys.Dev, 10) + "|" + strconv.FormatUint(sys.Ino, 10), nil
+	return strconv.FormatUint(sys.Dev, 10) + "|" + strconv.FormatUint(sys.Ino, 10),
+		uint64(sys.Nlink),
+		nil
 }