From 706eee3ae80e6a739e36a977b147f8565d310fa8 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Thu, 18 Jan 2024 02:57:05 +1300 Subject: [PATCH] Too many connections (#1586) * Single DB open for scrape functions & max pool size * Set idle timeout to avoid bad connection msgs * Rename db variable for clarity * Update paths.go --------- Co-authored-by: crwxaj <52156245+crwxaj@users.noreply.github.com> --- pkg/api/actors.go | 7 ++- pkg/common/common.go | 19 ++++---- pkg/common/paths.go | 7 +++ pkg/models/db.go | 35 +++++++++++++++ pkg/models/model_actor.go | 42 ++++++++---------- pkg/models/model_aka.go | 24 +++++------ pkg/models/model_external_reference.go | 60 +++++++++++--------------- pkg/models/model_scene.go | 59 +++++++++++-------------- pkg/scrape/badoink.go | 3 -- pkg/scrape/czechvr.go | 2 - pkg/scrape/genericactorscraper.go | 29 ++++++------- pkg/scrape/slrstudios.go | 7 +-- pkg/scrape/vrporn.go | 5 +-- pkg/tasks/content.go | 26 +++++------ 14 files changed, 157 insertions(+), 168 deletions(-) diff --git a/pkg/api/actors.go b/pkg/api/actors.go index f698f0a1b..2a28b4d3c 100644 --- a/pkg/api/actors.go +++ b/pkg/api/actors.go @@ -710,11 +710,10 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp var links []models.ExternalReferenceLink - db, _ := models.GetDB() - defer db.Close() + commonDb, _ := models.GetCommonDB() // find any links that were removed - db.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links) + commonDb.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links) for _, link := range links { found := false for _, url := range urls { @@ -724,7 +723,7 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp } } if !found { - db.Delete(&link) + commonDb.Delete(&link) models.AddActionActor(actor.ID, "edit_actor", "delete", "external_reference_link", link.ExternalReference.ExternalURL) } } diff --git a/pkg/common/common.go b/pkg/common/common.go index 972895de2..693b91486 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -12,15 +12,16 @@ var ( ) type EnvConfigSpec struct { - Debug bool `envconfig:"DEBUG" default:"false"` - DebugRequests bool `envconfig:"DEBUG_REQUESTS" default:"false"` - DebugSQL bool `envconfig:"DEBUG_SQL" default:"false"` - DebugWS bool `envconfig:"DEBUG_WS" default:"false"` - UIUsername string `envconfig:"UI_USERNAME" required:"false"` - UIPassword string `envconfig:"UI_PASSWORD" required:"false"` - DatabaseURL string `envconfig:"DATABASE_URL" required:"false" default:""` - WsAddr string `envconfig:"XBVR_WS_ADDR" required:"false" default:""` - WebPort int `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"` + Debug bool `envconfig:"DEBUG" default:"false"` + DebugRequests bool `envconfig:"DEBUG_REQUESTS" default:"false"` + DebugSQL bool `envconfig:"DEBUG_SQL" default:"false"` + DebugWS bool `envconfig:"DEBUG_WS" default:"false"` + UIUsername string `envconfig:"UI_USERNAME" required:"false"` + UIPassword string `envconfig:"UI_PASSWORD" required:"false"` + DatabaseURL string `envconfig:"DATABASE_URL" required:"false" default:""` + WsAddr string `envconfig:"XBVR_WS_ADDR" required:"false" default:""` + WebPort int `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"` + DBConnectionPoolSize int `envconfig:"DB_CONNECTION_POOL_SIZE" required:"false" default:"0"` } var EnvConfig EnvConfigSpec diff --git a/pkg/common/paths.go b/pkg/common/paths.go index b206c5132..7e5b258b0 100644 --- a/pkg/common/paths.go +++ b/pkg/common/paths.go @@ -23,6 +23,7 @@ var ScriptHeatmapDir string var MyFilesDir string var DownloadDir string var WebPort int +var DBConnectionPoolSize int func DirSize(path string) (int64, error) { var size int64 @@ -51,6 +52,7 @@ func InitPaths() { databaseurl := flag.String("database_url", "", "Optional: override default database path") web_port := flag.Int("web_port", 0, "Optional: override default Web Page port 9999") ws_addr := flag.String("ws_addr", "", "Optional: override default Websocket address from the default 0.0.0.0:9998") + db_connection_pool_size := flag.Int("db_connection_pool_size", 0, "Optional: sets a limit to the number of db connections while scraping") flag.Parse() @@ -113,6 +115,11 @@ func InitPaths() { WsAddr = EnvConfig.WsAddr } } + if *db_connection_pool_size != 0 { + DBConnectionPoolSize = *db_connection_pool_size + } else { + DBConnectionPoolSize = EnvConfig.DBConnectionPoolSize + } _ = os.MkdirAll(AppDir, os.ModePerm) _ = os.MkdirAll(ImgDir, os.ModePerm) diff --git a/pkg/models/db.go b/pkg/models/db.go index 341770afe..3927a6b9c 100644 --- a/pkg/models/db.go +++ b/pkg/models/db.go @@ -2,6 +2,7 @@ package models import ( "strings" + "time" "github.com/avast/retry-go/v4" "github.com/jinzhu/gorm" @@ -16,6 +17,7 @@ import ( var log = &common.Log var dbConn *dburl.URL var supportedDB = []string{"mysql", "sqlite3"} +var commonConnection *gorm.DB func parseDBConnString() { var err error @@ -78,6 +80,38 @@ func GetDB() (*gorm.DB, error) { return db, nil } +func GetCommonDB() (*gorm.DB, error) { + if common.EnvConfig.DebugSQL { + log.Debug("Getting Common DB handle from ", common.GetCallerFunctionName()) + } + + var err error + + if commonConnection != nil { + return commonConnection, nil + } + err = retry.Do( + func() error { + commonConnection, err = gorm.Open(dbConn.Driver, dbConn.DSN) + commonConnection.LogMode(common.EnvConfig.DebugSQL) + commonConnection.DB().SetConnMaxIdleTime(4 * time.Minute) + if common.DBConnectionPoolSize > 0 { + commonConnection.DB().SetMaxOpenConns(common.DBConnectionPoolSize) + } + if err != nil { + return err + } + return nil + }, + ) + + if err != nil { + log.Fatal("Failed to connect to database ", err) + } + + return commonConnection, nil +} + // Lock functions func CreateLock(lock string) { @@ -127,4 +161,5 @@ func init() { common.InitPaths() common.InitLogging() parseDBConnString() + GetCommonDB() } diff --git a/pkg/models/model_actor.go b/pkg/models/model_actor.go index f7c37d1b3..03ab9965f 100644 --- a/pkg/models/model_actor.go +++ b/pkg/models/model_actor.go @@ -97,12 +97,11 @@ type ActorLink struct { } func (i *Actor) Save() error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var err error = retry.Do( func() error { - err := db.Save(&i).Error + err := commonDb.Save(&i).Error if err != nil { return err } @@ -118,8 +117,7 @@ func (i *Actor) Save() error { } func (i *Actor) CountActorTags() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() type CountResults struct { ID int @@ -131,7 +129,7 @@ func (i *Actor) CountActorTags() { var results []CountResults - db.Model(&Actor{}). + commonDb.Model(&Actor{}). Select("actors.id, count as existingcnt, count(*) cnt, sum(scenes.is_available ) is_available, avail_count as existingavail"). Group("actors.id"). Joins("join scene_cast on scene_cast.actor_id = actors.id"). @@ -141,7 +139,7 @@ func (i *Actor) CountActorTags() { for i := range results { var actor Actor if results[i].Cnt != results[i].Existingcnt || results[i].IsAvailable != results[i].Existingavail { - db.First(&actor, results[i].ID) + commonDb.First(&actor, results[i].ID) actor.Count = results[i].Cnt actor.AvailCount = results[i].IsAvailable actor.Save() @@ -170,11 +168,10 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList { limit := r.Limit.OrElse(100) offset := r.Offset.OrElse(0) - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var actors []Actor - tx := db.Model(&actors) + tx := commonDb.Model(&actors) var out ResponseActorList @@ -504,10 +501,9 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList { } func (o *Actor) GetIfExist(id string) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db. + return commonDb. Preload("Scenes", func(db *gorm.DB) *gorm.DB { return db.Where("is_hidden = 0") }). @@ -515,10 +511,9 @@ func (o *Actor) GetIfExist(id string) error { } func (o *Actor) GetIfExistByPK(id uint) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db. + return commonDb. Preload("Scenes", func(db *gorm.DB) *gorm.DB { return db.Where("is_hidden = 0") }). @@ -526,10 +521,9 @@ func (o *Actor) GetIfExistByPK(id uint) error { } func (o *Actor) GetIfExistByPKWithSceneAvg(id uint) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - tx := db.Model(&Actor{}) + tx := commonDb.Model(&Actor{}) tx = tx.Select(`actors.*, (select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) as scene_rating_average`) @@ -631,19 +625,17 @@ func addToStringArray(inputArray string, newValue string) (string, bool) { func (a *Actor) CheckForSetImage() bool { // check if the field was deleted by the user, - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var action ActionActor - db.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action) + commonDb.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action) return action.ID != 0 } func (a *Actor) CheckForUserDeletes(fieldName string, newValue string) bool { // check if the field was deleted by the user, - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var action ActionActor - db.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action) + commonDb.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action) if action.ID != 0 && action.ActionType == "delete" { return true } diff --git a/pkg/models/model_aka.go b/pkg/models/model_aka.go index 35b44b45f..dc2533db4 100644 --- a/pkg/models/model_aka.go +++ b/pkg/models/model_aka.go @@ -20,12 +20,11 @@ type Aka struct { } func (i *Aka) Save() error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() err := retry.Do( func() error { - err := db.Save(&i).Error + err := commonDb.Save(&i).Error if err != nil { return err } @@ -41,24 +40,22 @@ func (i *Aka) Save() error { } func (o *Aka) GetIfExistByPK(id uint) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db. + return commonDb. Preload("Actors"). Where(&Aka{ID: id}).First(o).Error } func (o *Aka) UpdateAkaSceneCastRecords() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() // Queries to update the scene_cast table for the aka actor are comlex but fast. // Significating faster than iterating through the results of multiple simpler queries. // The Raw Sql used is compatible between mysql & sqlite // add missing scene_cast records for aka actors - db.Exec(` + commonDb.Exec(` insert into scene_cast select distinct sc.scene_id, a.aka_actor_id from akas a @@ -69,7 +66,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() { `) // delete scene_cast records for aka actors that have been removed - db.Exec(` + commonDb.Exec(` with SceneIds as ( select distinct a.id, sc.scene_id from akas a @@ -95,8 +92,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() { } func (o *Aka) RefreshAkaActorNames() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() type SortedList struct { AkaActorId uint @@ -105,7 +101,7 @@ func (o *Aka) RefreshAkaActorNames() { var sortedList []SortedList // this update the aka names by reordering the actor names based on descending count - db.Raw(` + commonDb.Raw(` with sorted as ( select a.aka_actor_id, a2.name, a2.count from akas a join actor_akas aa on a.id =aa.aka_id @@ -118,7 +114,7 @@ func (o *Aka) RefreshAkaActorNames() { for _, listItem := range sortedList { var actor Actor actor.ID = listItem.AkaActorId - db.Model(&actor).Where("name != ?", "aka:"+listItem.SortedName).Update("name", "aka:"+listItem.SortedName) + commonDb.Model(&actor).Where("name != ?", "aka:"+listItem.SortedName).Update("name", "aka:"+listItem.SortedName) } } diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index 9c779a3c3..8598e096a 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -90,33 +90,29 @@ type SceneMatchRule struct { } func (o *ExternalReference) GetIfExist(id uint) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db.Preload("XbvrLinks").Where(&ExternalReference{ID: id}).First(o).Error + return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ID: id}).First(o).Error } func (o *ExternalReference) FindExternalUrl(externalSource string, externalUrl string) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalURL: externalUrl}).First(o).Error + return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalURL: externalUrl}).First(o).Error } func (o *ExternalReference) FindExternalId(externalSource string, externalId string) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalId: externalId}).First(o).Error + return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalId: externalId}).First(o).Error } func (o *ExternalReference) Save() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() err := retry.Do( func() error { - err := db.Save(&o).Error + err := commonDb.Save(&o).Error if err != nil { return err } @@ -129,14 +125,12 @@ func (o *ExternalReference) Save() { } func (o *ExternalReference) Delete() { - db, _ := GetDB() - db.Delete(&o) - db.Close() + commonDb, _ := GetCommonDB() + commonDb.Delete(&o) } func (o *ExternalReference) AddUpdateWithUrl() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() existingRef := ExternalReference{ExternalSource: o.ExternalSource, ExternalURL: o.ExternalURL} existingRef.FindExternalUrl(o.ExternalSource, o.ExternalURL) @@ -153,7 +147,7 @@ func (o *ExternalReference) AddUpdateWithUrl() { err := retry.Do( func() error { - err := db.Save(&o).Error + err := commonDb.Save(&o).Error if err != nil { return err } @@ -166,8 +160,7 @@ func (o *ExternalReference) AddUpdateWithUrl() { } func (o *ExternalReference) AddUpdateWithId() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() existingRef := ExternalReference{ExternalSource: o.ExternalSource, ExternalId: o.ExternalId} existingRef.FindExternalId(o.ExternalSource, o.ExternalId) @@ -184,7 +177,7 @@ func (o *ExternalReference) AddUpdateWithId() { err := retry.Do( func() error { - err := db.Save(&o).Error + err := commonDb.Save(&o).Error if err != nil { return err } @@ -197,12 +190,11 @@ func (o *ExternalReference) AddUpdateWithId() { } func (o *ExternalReferenceLink) Save() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() err := retry.Do( func() error { - err := db.Save(&o).Error + err := commonDb.Save(&o).Error if err != nil { return err } @@ -215,10 +207,9 @@ func (o *ExternalReferenceLink) Save() { } func (o *ExternalReferenceLink) Find(externalSource string, internalName string) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db.Where(&ExternalReferenceLink{ExternalSource: externalSource, InternalNameId: internalName}).First(o).Error + return commonDb.Where(&ExternalReferenceLink{ExternalSource: externalSource, InternalNameId: internalName}).First(o).Error } func FormatInternalDbId(input uint) string { @@ -264,11 +255,10 @@ func (o *ExternalReference) DetermineActorScraperByUrl(url string) string { } func (o *ExternalReference) DetermineActorScraperBySiteId(siteId string) string { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var site Site - db.Where("id = ?", siteId).First(&site) + commonDb.Where("id = ?", siteId).First(&site) if site.Name == "" { return siteId } @@ -304,8 +294,7 @@ func (config ActorScraperConfig) loadActorScraperRules() { } func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var sites []Site // To understand the regex used, sign up to chat.openai.com and just ask something like Explain (.*, )?(.*)$ @@ -357,7 +346,7 @@ func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() { siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "aliases", Selector: `div[data-qa="model-info-aliases"] div.u-wh`}) scrapeRules.GenericActorScrapingConfig["slr-originals scrape"] = siteDetails scrapeRules.GenericActorScrapingConfig["slr-jav-originals scrape"] = siteDetails - db.Where("name like ?", "%SLR)").Find(&sites) + commonDb.Where("name like ?", "%SLR)").Find(&sites) scrapeRules.GenericActorScrapingConfig["slr scrape"] = siteDetails siteDetails = GenericScraperRuleSet{} @@ -1080,8 +1069,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() { } func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var sites []Site @@ -1191,7 +1179,7 @@ func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() { Rules: []SceneMatchRule{{XbvrField: "scene_url", XbvrMatch: `(lethalhardcorevr.com).*\/(\d{6,8})\/.*`, XbvrMatchResultPosition: 2, StashRule: `(lethalhardcorevr.com).*\/(\d{6,8})\/.*`, StashMatchResultPosition: 2}}, } - db.Where(&Site{IsEnabled: true}).Order("id").Find(&sites) + commonDb.Where(&Site{IsEnabled: true}).Order("id").Find(&sites) for _, site := range sites { if _, found := scrapeRules.StashSceneMatching[site.ID]; !found { if strings.HasSuffix(site.Name, "SLR)") { diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index 1ff13bb00..9ffcfd220 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -34,12 +34,11 @@ type SceneCuepoint struct { } func (o *SceneCuepoint) Save() error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var err error = retry.Do( func() error { - err := db.Save(&o).Error + err := commonDb.Save(&o).Error if err != nil { return err } @@ -133,12 +132,11 @@ type VideoSource struct { } func (i *Scene) Save() error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var err error = retry.Do( func() error { - err := db.Save(&i).Error + err := commonDb.Save(&i).Error if err != nil { return err } @@ -162,10 +160,9 @@ func (i *Scene) FromJSON(data []byte) error { } func (o *Scene) GetIfExist(id string) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db. + return commonDb. Preload("Tags"). Preload("Cast"). Preload("Files"). @@ -175,10 +172,9 @@ func (o *Scene) GetIfExist(id string) error { } func (o *Scene) GetIfExistByPK(id uint) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db. + return commonDb. Preload("Tags"). Preload("Cast"). Preload("Files"). @@ -188,10 +184,9 @@ func (o *Scene) GetIfExistByPK(id uint) error { } func (o *Scene) GetIfExistURL(u string) error { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() - return db. + return commonDb. Preload("Tags"). Preload("Cast"). Preload("Files"). @@ -215,21 +210,19 @@ func (o *Scene) GetFunscriptTitle() string { } func (o *Scene) GetFiles() ([]File, error) { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var files []File - db.Preload("Volume").Where(&File{SceneID: o.ID}).Find(&files) + commonDb.Preload("Volume").Where(&File{SceneID: o.ID}).Find(&files) return files, nil } func (o *Scene) GetTotalWatchTime() int { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() totalResult := struct{ Total float64 }{} - db.Raw(`select sum(duration) as total from histories where scene_id = ?`, o.ID).Scan(&totalResult) + commonDb.Raw(`select sum(duration) as total from histories where scene_id = ?`, o.ID).Scan(&totalResult) return int(totalResult.Total) } @@ -240,14 +233,13 @@ func (o *Scene) GetVideoFiles() ([]File, error) { } func (o *Scene) GetVideoFilesSorted(sort string) ([]File, error) { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var files []File if sort == "" { - db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Find(&files) + commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Find(&files) } else { - db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Order(sort).Find(&files) + commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "video").Order(sort).Find(&files) } return files, nil @@ -259,35 +251,32 @@ func (o *Scene) GetScriptFiles() ([]File, error) { } func (o *Scene) GetScriptFilesSorted(sort string) ([]File, error) { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var files []File if sort == "" { - db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Find(&files) + commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Find(&files) } else { - db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Order(sort).Find(&files) + commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "script").Order(sort).Find(&files) } return files, nil } func (o *Scene) GetHSPFiles() ([]File, error) { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var files []File - db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "hsp").Find(&files) + commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "hsp").Find(&files) return files, nil } func (o *Scene) GetSubtitlesFiles() ([]File, error) { - db, _ := GetDB() - defer db.Close() + commonDb, _ := GetCommonDB() var files []File - db.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Find(&files) + commonDb.Preload("Volume").Where("scene_id = ? AND type = ?", o.ID, "subtitles").Find(&files) return files, nil } diff --git a/pkg/scrape/badoink.go b/pkg/scrape/badoink.go index aa5760b10..981137772 100644 --- a/pkg/scrape/badoink.go +++ b/pkg/scrape/badoink.go @@ -31,9 +31,6 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out siteCollector := createCollector("badoinkvr.com", "babevr.com", "vrcosplayx.com", "18vr.com", "kinkvr.com") trailerCollector := cloneCollector(sceneCollector) - db, _ := models.GetDB() - defer db.Close() - sceneCollector.OnHTML(`html`, func(e *colly.HTMLElement) { sc := models.ScrapedScene{} sc.ScraperID = scraperID diff --git a/pkg/scrape/czechvr.go b/pkg/scrape/czechvr.go index 40c3b18a3..5729b825d 100644 --- a/pkg/scrape/czechvr.go +++ b/pkg/scrape/czechvr.go @@ -17,8 +17,6 @@ import ( func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, nwID string, singeScrapeAdditionalInfo string) error { defer wg.Done() logScrapeStart(scraperID, siteID) - db, _ := models.GetDB() - defer db.Close() sceneCollector := createCollector("www.czechvrnetwork.com") siteCollector := createCollector("www.czechvrnetwork.com") diff --git a/pkg/scrape/genericactorscraper.go b/pkg/scrape/genericactorscraper.go index 4b723ff3e..01327ea04 100644 --- a/pkg/scrape/genericactorscraper.go +++ b/pkg/scrape/genericactorscraper.go @@ -34,20 +34,19 @@ func GenericActorScrapers() { tlog := log.WithField("task", "scrape") tlog.Infof("Scraping Actor Details from Sites") - db, _ := models.GetDB() - defer db.Close() + commonDb, _ := models.GetCommonDB() scraperConfig := models.BuildActorScraperRules() var actors []models.Actor - db.Preload("Scenes"). + commonDb.Preload("Scenes"). Where("id =1"). Find(&actors) sqlcmd := "" var output []outputList - switch db.Dialect().GetName() { + switch commonDb.Dialect().GetName() { // gets the list of an actors Urls for scraper and join to external_reference_links to see if the haven't been linked case "mysql": sqlcmd = ` @@ -78,7 +77,7 @@ func GenericActorScrapers() { processed := 0 lastMessage := time.Now() - db.Raw(sqlcmd).Scan(&output) + commonDb.Raw(sqlcmd).Scan(&output) var wg sync.WaitGroup concurrentLimit := 10 // Maximum number of concurrent tasks @@ -133,16 +132,15 @@ func processAuthorLink(row outputList, siteRules map[string]models.GenericScrape func GenericSingleActorScraper(actorId uint, actorPage string) { log.Infof("Scraping Actor Details from %s", actorPage) - db, _ := models.GetDB() - defer db.Close() + commonDb, _ := models.GetCommonDB() var actor models.Actor actor.ID = actorId - db.Find(&actor) + commonDb.Find(&actor) scraperConfig := models.BuildActorScraperRules() var extRefLink models.ExternalReferenceLink - db.Preload("ExternalReference"). + commonDb.Preload("ExternalReference"). Where(&models.ExternalReferenceLink{ExternalId: actorPage, InternalDbId: actor.ID}). First(&extRefLink) @@ -160,16 +158,14 @@ func GenericActorScrapersBySite(site string) { tlog := log.WithField("task", "scrape") tlog.Infof("Scraping Actor Details from %s", site) - db, _ := models.GetDB() - defer db.Close() - + commonDb, _ := models.GetCommonDB() scraperConfig := models.BuildActorScraperRules() er := models.ExternalReference{} scrapeId := er.DetermineActorScraperBySiteId(site) var actors []models.Actor - db.Debug().Select("DISTINCT actors.*"). + commonDb.Debug().Select("DISTINCT actors.*"). Joins("JOIN scene_cast ON scene_cast.actor_id = actors.id"). Joins("JOIN scenes ON scenes.id = scene_cast.scene_id"). Where("scenes.scraper_id = ?", site). @@ -184,7 +180,7 @@ func GenericActorScrapersBySite(site string) { // find the url for the actor for this site var extreflink models.ExternalReferenceLink - db.Where(`internal_table = 'actors' and internal_db_id = ? and external_source = ?`, actor.ID, scrapeId).First(&extreflink) + commonDb.Where(`internal_table = 'actors' and internal_db_id = ? and external_source = ?`, actor.ID, scrapeId).First(&extreflink) for source, rule := range scraperConfig.GenericActorScrapingConfig { if source == scrapeId { applyRules(extreflink.ExternalId, scrapeId, rule, &actor, true) @@ -278,9 +274,8 @@ func applyRules(actorPage string, source string, rules models.GenericScraperRule var extref models.ExternalReference var extreflink models.ExternalReferenceLink - db, _ := models.GetDB() - defer db.Close() - db.Preload("ExternalReference"). + commonDb, _ := models.GetCommonDB() + commonDb.Preload("ExternalReference"). Where(&models.ExternalReferenceLink{ExternalSource: source, InternalDbId: actor.ID}). First(&extreflink) extref = extreflink.ExternalReference diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go index d6056f92c..cded1b813 100644 --- a/pkg/scrape/slrstudios.go +++ b/pkg/scrape/slrstudios.go @@ -22,8 +22,7 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out sceneCollector := createCollector("www.sexlikereal.com") siteCollector := createCollector("www.sexlikereal.com") - db, _ := models.GetDB() - defer db.Close() + commonDb, _ := models.GetCommonDB() // RegEx Patterns coverRegEx := regexp.MustCompile(`background(?:-image)?\s*?:\s*?url\s*?\(\s*?(.*?)\s*?\)`) @@ -48,10 +47,8 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out studioId = strings.TrimSuffix(strings.ReplaceAll(studioId, "/studios/", ""), "/") // see if we can find the site record, there may not be - db, _ := models.GetDB() - defer db.Close() var site models.Site - db.Where("id = ? or name like ? or (name = ? and name like 'SLR%')", studioId, sc.Studio+"%SLR)", sc.Studio).First(&site) + commonDb.Where("id = ? or name like ? or (name = ? and name like 'SLR%')", studioId, sc.Studio+"%SLR)", sc.Studio).First(&site) if site.ID != "" { sc.ScraperID = site.ID } diff --git a/pkg/scrape/vrporn.go b/pkg/scrape/vrporn.go index b8b7a78f3..a62ca1630 100644 --- a/pkg/scrape/vrporn.go +++ b/pkg/scrape/vrporn.go @@ -45,10 +45,9 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< sc.Studio = strings.TrimSpace(e.Text) sc.Site = sc.Studio // see if we can find the site record, there may not be - db, _ := models.GetDB() - defer db.Close() + commonDb, _ := models.GetCommonDB() var site models.Site - db.Where("name like ?", sc.Studio+"%VRPorn) or id = ?", sc.Studio, studioId).First(&site) + commonDb.Where("name like ?", sc.Studio+"%VRPorn) or id = ?", sc.Studio, studioId).First(&site) if site.ID != "" { sc.ScraperID = site.ID } diff --git a/pkg/tasks/content.go b/pkg/tasks/content.go index 4607e8a50..776db456c 100644 --- a/pkg/tasks/content.go +++ b/pkg/tasks/content.go @@ -104,15 +104,14 @@ func runScrapers(knownScenes []string, toScrape string, updateSite bool, collect scrapers := models.GetScrapers() var sites []models.Site - db, _ := models.GetDB() + commonDb, _ := models.GetCommonDB() if toScrape == "_all" { - db.Find(&sites) + commonDb.Find(&sites) } else if toScrape == "_enabled" { - db.Where(&models.Site{IsEnabled: true}).Find(&sites) + commonDb.Where(&models.Site{IsEnabled: true}).Find(&sites) } else { - db.Where(&models.Site{ID: toScrape}).Find(&sites) + commonDb.Where(&models.Site{ID: toScrape}).Find(&sites) } - db.Close() var wg sync.WaitGroup @@ -151,18 +150,17 @@ func sceneSliceAppender(collectedScenes *[]models.ScrapedScene, scenes <-chan mo func sceneDBWriter(wg *sync.WaitGroup, i *uint64, scenes <-chan models.ScrapedScene, processedScenes *[]models.ScrapedScene, lock *sync.Mutex) { defer wg.Done() - db, _ := models.GetDB() - defer db.Close() + commonDb, _ := models.GetCommonDB() for scene := range scenes { if os.Getenv("DEBUG") != "" { log.Printf("Saving %v", scene.SceneID) } if scene.OnlyUpdateScriptData { if config.Config.Funscripts.ScrapeFunscripts { - models.SceneUpdateScriptData(db, scene) + models.SceneUpdateScriptData(commonDb, scene) } } else { - models.SceneCreateUpdateFromExternal(db, scene) + models.SceneCreateUpdateFromExternal(commonDb, scene) } // Add the processed scene to the list to re/index lock.Lock() @@ -261,9 +259,8 @@ func ReapplyEdits() { func ScrapeSingleScene(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo string) models.Scene { var newScene models.Scene Scrape(toScrape, singleSceneURL, singeScrapeAdditionalInfo) - db, _ := models.GetDB() - defer db.Close() - db. + commonDb, _ := models.GetCommonDB() + commonDb. Preload("Tags"). Preload("Cast"). Preload("Files"). @@ -286,9 +283,8 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st // Get all known scenes var scenes []models.Scene - db, _ := models.GetDB() - db.Find(&scenes) - db.Close() + commonDb, _ := models.GetCommonDB() + commonDb.Find(&scenes) var knownScenes []string for i := range scenes {