From 10bfaec2e968083e7475672e55a4c35f40b972bc Mon Sep 17 00:00:00 2001 From: vt-idiot <81622808+vt-idiot@users.noreply.github.com> Date: Sun, 5 Nov 2023 08:11:07 -0500 Subject: [PATCH] scraper: Better filename generation for SLR scrapers (#1440) * Update slrstudios.go Attempt at re-writing the filename part of the scraper. * Update slrstudios.go Uses the API to properly set only the relevant filenames. * Update slrstudios.go Additional fix for trans scene ID collisions. And the old filename method is necessary for trans scenes unless someone knows the API endpoint. --- pkg/scrape/slrstudios.go | 127 +++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 39 deletions(-) diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go index 4c0b2f353..4ecd26593 100644 --- a/pkg/scrape/slrstudios.go +++ b/pkg/scrape/slrstudios.go @@ -133,6 +133,10 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out sc.TrailerType = "slr" sc.TrailerSrc = "https://api.sexlikereal.com/virtualreality/video/id/" + sc.SiteID + s, _ := resty.New().R(). + SetHeader("User-Agent", UserAgent). + Get(sc.TrailerSrc) + JsonMetadataA := s.String() isTransScene := e.Request.Ctx.GetAny("isTransScene").(bool) @@ -202,6 +206,9 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out // Title sc.Title = gjson.Get(JsonMetadata, "title").String() + // Fix Scene ID Collisions + sc.SceneID = "slr-trans-" + sc.SiteID + // Duration - Not Available // Filenames @@ -223,10 +230,6 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out // Passthrough "chromaKey":{"enabled":false,"hasAlpha":true,"h":0,"opacity":1,"s":0,"threshold":0,"v":0} if alphA == "PT" { - s, _ := resty.New().R(). - SetHeader("User-Agent", UserAgent). - Get(sc.TrailerSrc) - JsonMetadataA := s.String() if gjson.Get(JsonMetadataA, "chromaKey").Exists() { sc.ChromaKey = gjson.Get(JsonMetadataA, "chromaKey").String() } @@ -234,7 +237,7 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out } // Filenames - appendFilenames(&sc, siteID, filenameRegEx, videotype, FB360, alphA) + appendFilenames(&sc, siteID, filenameRegEx, videotype, FB360, alphA, JsonMetadataA, isTransScene) // actor details sc.ActorDetails = make(map[string]models.ActorDetails) @@ -355,47 +358,93 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out return nil } -func appendFilenames(sc *models.ScrapedScene, siteID string, filenameRegEx *regexp.Regexp, videotype string, FB360 string, AlphA string) { +func appendFilenames(sc *models.ScrapedScene, siteID string, filenameRegEx *regexp.Regexp, videotype string, FB360 string, AlphA string, JsonMetadataA string, isTransScene bool) { // Only shown for logged in users so need to generate them // Format: SLR_siteID_Title__SceneID__<180/360>.mp4 - resolutions := []string{"_6400p_", "_4096p_", "_4000p_", "_3840p_", "_3360p_", "_3160p_", "_3072p_", "_3000p_", "_2900p_", "_2880p_", "_2700p_", "_2650p_", "_2160p_", "_1920p_", "_1440p_", "_1080p_", "_original_"} - baseName := "SLR_" + strings.TrimSuffix(siteID, " (SLR)") + "_" + filenameRegEx.ReplaceAllString(sc.Title, "_") - switch videotype { - case "360°": // Sadly can't determine if TB or MONO so have to add both - for i := range resolutions { - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MONO_360.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_TB_360.mp4") - } - case "Fisheye": // 200° videos named with MKX200 - for i := range resolutions { + if !isTransScene { + viewAngle := gjson.Get(JsonMetadataA, "viewAngle").String() + projSuffix := "_LR_180.mp4" + if viewAngle == "190" || viewAngle == "200" || viewAngle == "220" { + screentype := strings.ToUpper(gjson.Get(JsonMetadataA, "screenType").String()) + projSuffix = "_" + screentype if AlphA == "true" { - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX200_alpha.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX220_alpha.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_RF52_alpha.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_FISHEYE190_alpha.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_VRCA220_alpha.mp4") + projSuffix = "_" + screentype + "_alpha" + } + if FB360 != "" { + FB360 = projSuffix + "_FB360.mkv" + } + projSuffix = projSuffix + ".mp4" + } else if viewAngle == "360" { + monotb := gjson.Get(JsonMetadataA, "stereomode").String() + if monotb == "mono" { + projSuffix = "_MONO_360.mp4" } else { - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX200.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX220.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_RF52.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_FISHEYE190.mp4") - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_VRCA220.mp4") + projSuffix = "_TB_360.mp4" } } - default: // Assuming everything else is 180 and LR, yet to find a TB_180 - for i := range resolutions { - sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_LR_180.mp4") + resolutions := []string{"_original_"} + encodings := gjson.Get(JsonMetadataA, "encodings.#(name=h265).videoSources.#.resolution") + for _, name := range encodings.Array() { + resolutions = append(resolutions, "_"+name.String()+"p_") + } + baseName := "SLR_" + strings.TrimSuffix(siteID, " (SLR)") + "_" + filenameRegEx.ReplaceAllString(sc.Title, "_") + switch videotype { + case "360°": + for i := range resolutions { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+projSuffix) + } + case "Fisheye": // 200° videos named with MKX200 + for i := range resolutions { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+projSuffix) + } + default: // Assuming everything else is 180 and LR, yet to find a TB_180 + for i := range resolutions { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+projSuffix) + } + } + if FB360 != "" { + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+FB360) + } + } else { + resolutions := []string{"_6400p_", "_4096p_", "_4000p_", "_3840p_", "_3360p_", "_3160p_", "_3072p_", "_3000p_", "_2900p_", "_2880p_", "_2700p_", "_2650p_", "_2160p_", "_1920p_", "_1440p_", "_1080p_", "_original_"} + baseName := "SLR_" + strings.TrimSuffix(siteID, " (SLR)") + "_" + filenameRegEx.ReplaceAllString(sc.Title, "_") + switch videotype { + case "360°": // Sadly can't determine if TB or MONO so have to add both + for i := range resolutions { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MONO_360.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_TB_360.mp4") + } + case "Fisheye": // 200° videos named with MKX200 + for i := range resolutions { + if AlphA == "true" { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX200_alpha.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX220_alpha.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_RF52_alpha.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_FISHEYE190_alpha.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_VRCA220_alpha.mp4") + } else { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX200.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_MKX220.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_RF52.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_FISHEYE190.mp4") + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_VRCA220.mp4") + } + } + default: // Assuming everything else is 180 and LR, yet to find a TB_180 + for i := range resolutions { + sc.Filenames = append(sc.Filenames, baseName+resolutions[i]+sc.SiteID+"_LR_180.mp4") + } + } + if FB360 != "" { + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_LR_180"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_MKX200"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_MKX220"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_RF52"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"FISHEYE190"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_VRCA220"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_MONO_360"+FB360) + sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_TB_360"+FB360) } - } - if FB360 != "" { - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_LR_180"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_MKX200"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_MKX220"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_RF52"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"FISHEYE190"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_VRCA220"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_MONO_360"+FB360) - sc.Filenames = append(sc.Filenames, baseName+"_original_"+sc.SiteID+"_TB_360"+FB360) } }