Feat: File matching with stash hashs (xbapps#1587)

* Lookup StashDB Ohash in File Matching

* Create Option

pkg/api/options.go

@@ -178,6 +178,14 @@ type RequestSCustomSiteCreate struct {
 	Company string `json:"scraperCompany"`
 }
 
+type GetStorageResponse struct {
+	Volumes    []models.Volume `json:"volumes"`
+	MatchOhash bool            `json:"match_ohash"`
+}
+type RequestSaveOptionsStorage struct {
+	MatchOhash bool `json:"match_ohash"`
+}
+
 type ConfigResource struct{}
 
 func (i ConfigResource) WebService() *restful.WebService {
@@ -228,6 +236,9 @@ func (i ConfigResource) WebService() *restful.WebService {
 		Param(ws.PathParameter("storage-id", "Storage ID").DataType("int")).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
 
+	ws.Route(ws.PUT("/storage").To(i.saveOptionsStorage).
+		Metadata(restfulspec.KeyOpenAPITags, tags))
+
 	// "DLNA" section endpoints
 	ws.Route(ws.PUT("/interface/dlna").To(i.saveOptionsDLNA).
 		Metadata(restfulspec.KeyOpenAPITags, tags))
@@ -478,7 +489,10 @@ func (i ConfigResource) listStorage(req *restful.Request, resp *restful.Response
        	(select sum(files.size) from files where files.volume_id = volumes.id) as total_size
 		from volumes order by last_scan desc;`).Scan(&vol)
 
-	resp.WriteHeaderAndEntity(http.StatusOK, vol)
+	var out GetStorageResponse
+	out.Volumes = vol
+	out.MatchOhash = config.Config.Storage.MatchOhash
+	resp.WriteHeaderAndEntity(http.StatusOK, out)
 }
 
 func (i ConfigResource) addStorage(req *restful.Request, resp *restful.Response) {
@@ -950,3 +964,16 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
 
 	resp.WriteHeader(http.StatusOK)
 }
+func (i ConfigResource) saveOptionsStorage(req *restful.Request, resp *restful.Response) {
+	var r RequestSaveOptionsStorage
+	err := req.ReadEntity(&r)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	config.Config.Storage.MatchOhash = r.MatchOhash
+	config.SaveConfig()
+
+	resp.WriteHeaderAndEntity(http.StatusOK, r)
+}

pkg/config/config.go

@@ -153,6 +153,9 @@ type ObjectConfig struct {
 			RunAtStartDelay int  `default:"0" json:"runAtStartDelay"`
 		} `json:"stashdbRescrapeSchedule"`
 	} `json:"cron"`
+	Storage struct {
+		MatchOhash bool `default:"false" json:"match_ohash"`
+	} `json:"storage"`
 }
 
 var (

pkg/scrape/stashdb.go

@@ -145,7 +145,7 @@ func findStudio(studio string, field string) FindStudioResult {
 	// Define the variables needed for your query as a Go map
 	variables := `{"` + field + `": "` + studio + `"}`
 
-	resp := callStashDb(query, variables)
+	resp := CallStashDb(query, variables)
 	var data FindStudioResult
 	json.Unmarshal(resp, &data)
 	return data
@@ -191,7 +191,7 @@ func getPerformersPage(studioId string, page int) QueryPerformerResult {
 		}
 		`
 
-	resp := callStashDb(query, variables)
+	resp := CallStashDb(query, variables)
 	var data QueryPerformerResult
 	json.Unmarshal(resp, &data)
 	return data
@@ -214,7 +214,7 @@ func getScenes(studioId string, parentId string, tagId string) QueryScenesResult
 	} else {
 		variables = getStudioSceneQueryVariable(studioId, page, count)
 	}
-	sceneList = getScenePage(variables)
+	sceneList = GetScenePage(variables)
 	nextList = sceneList
 	for len(nextList.Data.QueryScenes.Scenes) > 0 &&
 		len(sceneList.Data.QueryScenes.Scenes) < sceneList.Data.QueryScenes.Count && // {
@@ -225,7 +225,7 @@ func getScenes(studioId string, parentId string, tagId string) QueryScenesResult
 		} else {
 			variables = getStudioSceneQueryVariable(studioId, page, count)
 		}
-		nextList = getScenePage(variables)
+		nextList = GetScenePage(variables)
 		sceneList.Data.QueryScenes.Scenes = append(sceneList.Data.QueryScenes.Scenes, nextList.Data.QueryScenes.Scenes...)
 	}
 	return sceneList
@@ -267,7 +267,7 @@ func getParentSceneQueryVariable(parentId string, tagId string, page int, count
 }
 
 // calls graphql scene query and return a list of scenes
-func getScenePage(variables string) QueryScenesResult {
+func GetScenePage(variables string) QueryScenesResult {
 	query := `
 	query  queryScenes($input: SceneQueryInput!) {
 		queryScenes(input: $input) {
@@ -325,7 +325,7 @@ func getScenePage(variables string) QueryScenesResult {
 	  `
 
 	// Define the variables needed for your query as a Go map
-	resp := callStashDb(query, variables)
+	resp := CallStashDb(query, variables)
 	var data QueryScenesResult
 	json.Unmarshal(resp, &data)
 	return data
@@ -492,14 +492,15 @@ func getStashPerformer(performer string) FindPerformerResult {
 	// Define the variables needed for your query as a Go map
 	var data FindPerformerResult
 	variables := `{"id": "` + performer + `"}`
-	resp := callStashDb(query, variables)
+	resp := CallStashDb(query, variables)
 	err := json.Unmarshal(resp, &data)
 	if err != nil {
 		log.Errorf("Eror extracting actor json")
 	}
 	return data
 }
-func callStashDb(query string, rawVariables string) []byte {
+
+func CallStashDb(query string, rawVariables string) []byte {
 	var variables map[string]interface{}
 	json.Unmarshal([]byte(rawVariables), &variables)
 

pkg/tasks/volume.go

@@ -19,8 +19,10 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/thoas/go-funk"
 	"github.com/xbapps/xbvr/pkg/common"
+	"github.com/xbapps/xbvr/pkg/config"
 	"github.com/xbapps/xbvr/pkg/ffprobe"
 	"github.com/xbapps/xbvr/pkg/models"
+	"github.com/xbapps/xbvr/pkg/scrape"
 )
 
 var allowedVideoExt = []string{".mp4", ".avi", ".wmv", ".mpeg4", ".mov", ".mkv"}
@@ -84,6 +86,50 @@ func RescanVolumes(id int) {
 				files[i].SceneID = scenes[0].ID
 				files[i].Save()
 				scenes[0].UpdateStatus()
+			} else {
+				if config.Config.Storage.MatchOhash && config.Config.Advanced.StashApiKey != "" {
+					hash := files[i].OsHash
+					if len(hash) < 16 {
+						// the has in xbvr is sometiomes < 16 pad with zeros
+						paddingLength := 16 - len(hash)
+						hash = strings.Repeat("0", paddingLength) + hash
+					}
+					queryVariable := `
+				{"input":{
+					"fingerprints": {					
+						"value": "` + hash + `",
+						"modifier": "INCLUDES"
+					},				
+					"page": 1
+				}
+				}`
+					// call Stashdb graphql searching for os_hash
+					stashMatches := scrape.GetScenePage(queryVariable)
+					for _, match := range stashMatches.Data.QueryScenes.Scenes {
+						if match.ID != "" {
+							var externalRefLink models.ExternalReferenceLink
+							db.Where(&models.ExternalReferenceLink{ExternalSource: "stashdb scene", ExternalId: match.ID}).First(&externalRefLink)
+							if externalRefLink.ID != 0 {
+								files[i].SceneID = externalRefLink.InternalDbId
+								files[i].Save()
+								var scene models.Scene
+								scene.GetIfExistByPK(externalRefLink.InternalDbId)
+
+								// add filename tyo the array
+								var pfTxt []string
+								json.Unmarshal([]byte(scene.FilenamesArr), &pfTxt)
+								pfTxt = append(pfTxt, files[i].Filename)
+								tmp, _ := json.Marshal(pfTxt)
+								scene.FilenamesArr = string(tmp)
+								scene.Save()
+								models.AddAction(scene.SceneID, "match", "filenames_arr", scene.FilenamesArr)
+
+								scene.UpdateStatus()
+								log.Infof("File %s matched to Scene %s matched using stashdb hash %s", path.Base(files[i].Filename), scene.SceneID, hash)
+							}
+						}
+					}
+				}
 			}
 
 			if (i % 50) == 0 {

ui/src/store/optionsStorage.js

@@ -1,16 +1,26 @@
 import ky from 'ky'
 
 const state = {
-  items: []
+  items: [],
+  options: {
+    match_ohash: false,
+  },  
 }
 
 const mutations = {
 }
 
 const actions = {
   async load ({ state }, params) {
-    state.items = await ky.get('/api/options/storage').json()
-  }
+    await ky.get('/api/options/storage').json()
+    .then(data => {
+      state.items = data.volumes
+      state.options.match_ohash = data.match_ohash
+    })
+  },
+  async save ({ state }, enabled) { 
+    ky.put('/api/options/storage', { json: { ...state.options } })      
+  },  
 }
 
 export default {

ui/src/views/options/sections/Storage.vue

@@ -112,6 +112,22 @@
       </div>
     </div>
 
+    <hr/>
+
+  <div>
+    <h3 class="title">{{ $t('Options') }}</h3>
+    <b-field>
+      <b-switch v-model="match_ohash" type="is-default">
+        Match StashDB Hashes
+      </b-switch>
+    </b-field>
+    <b-field>
+      <b-button type="is-primary" @click="save">Save options</b-button>
+    </b-field>
+
+
+  </div>
+
   </div>
 
 </template>
@@ -161,9 +177,20 @@ export default {
     },
     rescanFolder: function (folder) {
       ky.get(`/api/task/rescan/${folder.id}`)
-    }
+    },
+    save () {
+      this.$store.dispatch('optionsStorage/save')
+    },
   },
   computed: {
+    match_ohash: {
+      get () {        
+        return this.$store.state.optionsStorage.options.match_ohash
+      },
+      set (value) {
+        this.$store.state.optionsStorage.options.match_ohash = value
+      },
+    },
     total () {
       let files = 0; let unmatched = 0; let size = 0
       this.$store.state.optionsStorage.items.map(v => {