Skip to content

Commit

Permalink
Feat: Match Scenes to Alternate Sources (xbapps#1599)
Browse files Browse the repository at this point in the history
  • Loading branch information
@toshski
toshski authored Jan 24, 2024
1 parent 92c5193 commit b80afdb
Show file tree
Hide file tree
Showing 43 changed files with 2,180 additions and 246 deletions.
128 changes: 127 additions & 1 deletion pkg/api/external_references.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,26 @@
package api

import (
"net/http"
"time"

restfulspec "github.com/emicklei/go-restful-openapi/v2"
"github.com/emicklei/go-restful/v3"
"github.com/xbapps/xbvr/pkg/models"
)

//var RequestBody []byte
// var RequestBody []byte
type RequestEditExtRefLink struct {
ID uint `json:"id"`
ExternalReferenceID uint `json:"external_reference_id"`
ExternalSource string `json:"external_source"`
ExternalId string `json:"external_id"`
MatchType int `json:"match_type"`
InternalTable string `json:"internal_table"`
InternalDbId uint `json:"internal_db_id"`
InternalNameId string `json:"internal_name_id"`
DeleteDate time.Time `json:"delete_date"`
}

type ExternalReference struct{}

Expand Down Expand Up @@ -39,5 +54,116 @@ func (i ExternalReference) WebService() *restful.WebService {
Metadata(restfulspec.KeyOpenAPITags, tags))
ws.Route(ws.GET("/generic/scrape_by_site/{site-id}").To(i.genericActorScraperBySite).
Metadata(restfulspec.KeyOpenAPITags, tags))
ws.Route(ws.POST("/edit_link").To(i.editExtRefLink).
Metadata(restfulspec.KeyOpenAPITags, tags).
Writes())
ws.Route(ws.DELETE("/delete_extref").To(i.deleteExtRefLink).
Metadata(restfulspec.KeyOpenAPITags, tags).
Writes())
ws.Route(ws.DELETE("/delete_extref_source").To(i.deleteExtRefSource).
Metadata(restfulspec.KeyOpenAPITags, tags).
Writes())
ws.Route(ws.DELETE("/delete_extref_source_links/all").To(i.deleteExtRefSourceLinks).
Metadata(restfulspec.KeyOpenAPITags, tags).
Writes())
ws.Route(ws.DELETE("/delete_extref_source_links/keep_manual").To(i.deleteExtRefSourceLinksKeepManualMatches).
Metadata(restfulspec.KeyOpenAPITags, tags).
Writes())
return ws
}

func (i ExternalReference) editExtRefLink(req *restful.Request, resp *restful.Response) {
var r RequestEditExtRefLink
err := req.ReadEntity(&r)
if err != nil {
log.Error(err)
return
}

var extreflink models.ExternalReferenceLink
if r.ID > 0 {
extreflink.ExternalReference.GetIfExist(r.ID)
} else {
extreflink.FindByExternaID(r.ExternalSource, r.ExternalId)
}
extreflink.InternalTable = r.InternalTable
extreflink.InternalDbId = r.InternalDbId
extreflink.InternalNameId = r.InternalNameId
extreflink.MatchType = r.MatchType
extreflink.Save()
resp.WriteHeaderAndEntity(http.StatusOK, extreflink)
}
func (i ExternalReference) deleteExtRefLink(req *restful.Request, resp *restful.Response) {
// delete a single external_reference_link
var r RequestEditExtRefLink
err := req.ReadEntity(&r)
if err != nil {
log.Error(err)
return
}

var extreflink models.ExternalReferenceLink
if r.ID > 0 {
extreflink.ExternalReference.GetIfExist(r.ID)
} else {
extreflink.FindByExternaID(r.ExternalSource, r.ExternalId)
}
extreflink.ExternalReference.Delete()
extreflink.Delete()
resp.WriteHeaderAndEntity(http.StatusOK, nil)
}
func (i ExternalReference) deleteExtRefSource(req *restful.Request, resp *restful.Response) {
// deletes all external_reference_links and external_references for a source
var r RequestEditExtRefLink
err := req.ReadEntity(&r)
if err != nil {
log.Error(err)
return
}
commonDb, _ := models.GetCommonDB()

commonDb.Where("external_source = ?", r.ExternalSource).Delete(models.ExternalReferenceLink{})
commonDb.Where("external_source = ?", r.ExternalSource).Delete(models.ExternalReference{})

resp.WriteHeaderAndEntity(http.StatusOK, nil)
}
func (i ExternalReference) deleteExtRefSourceLinks(req *restful.Request, resp *restful.Response) {
// deletes external_reference_links for a source
var r RequestEditExtRefLink
err := req.ReadEntity(&r)
if err != nil {
log.Error(err)
return
}
commonDb, _ := models.GetCommonDB()
commonDb.Where("external_source like ?", r.ExternalSource).Delete(models.ExternalReferenceLink{})

resp.WriteHeaderAndEntity(http.StatusOK, nil)
}
func (i ExternalReference) deleteExtRefSourceLinksKeepManualMatches(req *restful.Request, resp *restful.Response) {
// deletes external_reference_links for a source, but keeps links the user has manually set, ie match_type = 99999
var r RequestEditExtRefLink
err := req.ReadEntity(&r)
if err != nil {
log.Error(err)
return
}
db, _ := models.GetDB()
defer db.Close()

if r.DeleteDate.IsZero() {
db.Where("external_source like ? and match_type not in (99999, -1)", r.ExternalSource).Delete(models.ExternalReferenceLink{})
} else {
// Fetch records to delete
var recordsToDelete []models.ExternalReferenceLink
db.Debug().Joins("JOIN external_references ON external_reference_links.external_reference_id = external_references.id").
Where("external_reference_links.external_source LIKE ? AND match_type NOT IN (99999, -1) AND external_references.external_date >= ?", r.ExternalSource, r.DeleteDate).
Find(&recordsToDelete)
for _, record := range recordsToDelete {
db.Debug().Delete(&record)
}

}

resp.WriteHeaderAndEntity(http.StatusOK, nil)
}
101 changes: 90 additions & 11 deletions pkg/api/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,16 @@ type RequestSaveOptionsWeb struct {
}

type RequestSaveOptionsAdvanced struct {
ShowInternalSceneId bool `json:"showInternalSceneId"`
ShowHSPApiLink bool `json:"showHSPApiLink"`
ShowSceneSearchField bool `json:"showSceneSearchField"`
StashApiKey string `json:"stashApiKey"`
ScrapeActorAfterScene bool `json:"scrapeActorAfterScene"`
UseImperialEntry bool `json:"useImperialEntry"`
ShowInternalSceneId bool `json:"showInternalSceneId"`
ShowHSPApiLink bool `json:"showHSPApiLink"`
ShowSceneSearchField bool `json:"showSceneSearchField"`
StashApiKey string `json:"stashApiKey"`
ScrapeActorAfterScene bool `json:"scrapeActorAfterScene"`
UseImperialEntry bool `json:"useImperialEntry"`
LinkScenesAfterSceneScraping bool `json:"linkScenesAfterSceneScraping"`
UseAltSrcInFileMatching bool `json:"useAltSrcInFileMatching"`
UseAltSrcInScriptFilters bool `json:"useAltSrcInScriptFilters"`
IgnoreReleasedBefore time.Time `json:"ignoreReleasedBefore"`
}

type RequestSaveOptionsFunscripts struct {
Expand Down Expand Up @@ -165,17 +169,30 @@ type RequestSaveOptionsTaskSchedule struct {
StashdbRescrapeHourStart int `json:"stashdbRescrapeHourStart"`
StashdbRescrapeHourEnd int `json:"stashdbRescrapeHourEnd"`
StashdbRescrapeStartDelay int `json:"stashdbRescrapeStartDelay"`

LinkScenesEnabled bool `json:"linkScenesEnabled"`
LinkScenesHourInterval int `json:"linkScenesHourInterval"`
LinkScenesUseRange bool `json:"linkScenesUseRange"`
LinkScenesMinuteStart int `json:"linkScenesMinuteStart"`
LinkScenesHourStart int `json:"linkScenesHourStart"`
LinkScenesHourEnd int `json:"linkScenesHourEnd"`
LinkScenesStartDelay int `json:"linkScenesStartDelay"`
}
type RequestSaveSiteMatchParams struct {
SiteId string `json:"site"`
MatchParams models.AltSrcMatchParams `json:"match_params"`
}

type RequestCuepointsResponse struct {
Positions []string `json:"positions"`
Actions []string `json:"actions"`
}
type RequestSCustomSiteCreate struct {
Url string `json:"scraperUrl"`
Name string `json:"scraperName"`
Avatar string `json:"scraperAvatar"`
Company string `json:"scraperCompany"`
Url string `json:"scraperUrl"`
Name string `json:"scraperName"`
Avatar string `json:"scraperAvatar"`
Company string `json:"scraperCompany"`
MasterSiteId string `json:"masterSiteId"`
}

type GetStorageResponse struct {
Expand Down Expand Up @@ -225,6 +242,11 @@ func (i ConfigResource) WebService() *restful.WebService {
ws.Route(ws.POST("/scraper/delete-scenes").To(i.deleteScenes).
Metadata(restfulspec.KeyOpenAPITags, tags))

ws.Route(ws.GET("/site/match_params/{site}").To(i.siteMatchParams).
Metadata(restfulspec.KeyOpenAPITags, tags))
ws.Route(ws.POST("/site/save_match_params").To(i.saveSiteMatchParams).
Metadata(restfulspec.KeyOpenAPITags, tags))

// "Storage" section endpoints
ws.Route(ws.GET("/storage").To(i.listStorage).
Metadata(restfulspec.KeyOpenAPITags, tags))
Expand Down Expand Up @@ -382,6 +404,49 @@ func (i ConfigResource) listSitesWithDB(req *restful.Request, resp *restful.Resp
resp.WriteHeaderAndEntity(http.StatusOK, sites)
}

func (i ConfigResource) siteMatchParams(req *restful.Request, resp *restful.Response) {
db, _ := models.GetDB()
defer db.Close()

id := req.PathParameter("site")
if id == "" {
return
}

var site models.Site
err := site.GetIfExist(id)
if err != nil {
log.Error(err)
return
}

var matchParams models.AltSrcMatchParams
matchParams.UnmarshalParams(site.MatchingParams)
resp.WriteHeaderAndEntity(http.StatusOK, matchParams)
}
func (i ConfigResource) saveSiteMatchParams(req *restful.Request, resp *restful.Response) {
db, _ := models.GetDB()
defer db.Close()
var r RequestSaveSiteMatchParams
if err := req.ReadEntity(&r); err != nil {
APIError(req, resp, http.StatusInternalServerError, err)
return
}

var site models.Site
err := site.GetIfExist(r.SiteId)
if err != nil {
log.Error(err)
return
}

json, _ := json.Marshal(r.MatchParams)
site.MatchingParams = string(json)
site.Save()

resp.WriteHeaderAndEntity(http.StatusOK, nil)
}

func (i ConfigResource) saveOptionsWeb(req *restful.Request, resp *restful.Response) {
var r RequestSaveOptionsWeb
err := req.ReadEntity(&r)
Expand Down Expand Up @@ -425,6 +490,10 @@ func (i ConfigResource) saveOptionsAdvanced(req *restful.Request, resp *restful.
config.Config.Advanced.StashApiKey = r.StashApiKey
config.Config.Advanced.ScrapeActorAfterScene = r.ScrapeActorAfterScene
config.Config.Advanced.UseImperialEntry = r.UseImperialEntry
config.Config.Advanced.LinkScenesAfterSceneScraping = r.LinkScenesAfterSceneScraping
config.Config.Advanced.UseAltSrcInFileMatching = r.UseAltSrcInFileMatching
config.Config.Advanced.UseAltSrcInScriptFilters = r.UseAltSrcInScriptFilters
config.Config.Advanced.IgnoreReleasedBefore = r.IgnoreReleasedBefore
config.SaveConfig()

resp.WriteHeaderAndEntity(http.StatusOK, r)
Expand Down Expand Up @@ -873,6 +942,14 @@ func (i ConfigResource) saveOptionsTaskSchedule(req *restful.Request, resp *rest
config.Config.Cron.StashdbRescrapeSchedule.HourEnd = r.StashdbRescrapeHourEnd
config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay = r.StashdbRescrapeStartDelay

config.Config.Cron.LinkScenesSchedule.Enabled = r.LinkScenesEnabled
config.Config.Cron.LinkScenesSchedule.HourInterval = r.LinkScenesHourInterval
config.Config.Cron.LinkScenesSchedule.UseRange = r.LinkScenesUseRange
config.Config.Cron.LinkScenesSchedule.MinuteStart = r.LinkScenesMinuteStart
config.Config.Cron.LinkScenesSchedule.HourStart = r.LinkScenesHourStart
config.Config.Cron.LinkScenesSchedule.HourEnd = r.LinkScenesHourEnd
config.Config.Cron.LinkScenesSchedule.RunAtStartDelay = r.LinkScenesStartDelay

config.SaveConfig()

resp.WriteHeaderAndEntity(http.StatusOK, r)
Expand Down Expand Up @@ -908,6 +985,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
r.Name = strings.TrimSpace(r.Name)
r.Company = strings.TrimSpace(r.Company)
r.Avatar = strings.TrimSpace(r.Avatar)
r.MasterSiteId = strings.TrimSpace(r.MasterSiteId)
if r.Company == "" {
r.Company = r.Name
}
Expand Down Expand Up @@ -937,12 +1015,13 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
scrapers[key][idx].Name = r.Name
scrapers[key][idx].Company = r.Company
scrapers[key][idx].AvatarUrl = r.Avatar
scrapers[key][idx].MasterSiteId = r.MasterSiteId
}
}
}

if !exists {
scraper := config.ScraperConfig{URL: r.Url, Name: r.Name, Company: r.Company, AvatarUrl: r.Avatar}
scraper := config.ScraperConfig{URL: r.Url, Name: r.Name, Company: r.Company, AvatarUrl: r.Avatar, MasterSiteId: r.MasterSiteId}
switch match[3] {
case "povr":
scrapers["povr"] = append(scrapers["povr"], scraper)
Expand Down
Loading

0 comments on commit b80afdb

Please sign in to comment.