Too many connections (xbapps#1586)

* Single DB open for scrape functions & max pool size

* Set idle timeout to avoid bad connection msgs

* Rename db variable for clarity

* Update paths.go

---------

Co-authored-by: crwxaj <52156245+crwxaj@users.noreply.github.com>

pkg/api/actors.go

@@ -710,11 +710,10 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
 
 	var links []models.ExternalReferenceLink
 
-	db, _ := models.GetDB()
-	defer db.Close()
+	commonDb, _ := models.GetCommonDB()
 
 	// find any links that were removed
-	db.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links)
+	commonDb.Preload("ExternalReference").Where("internal_table = 'actors' and internal_db_id = ?", id).Find(&links)
 	for _, link := range links {
 		found := false
 		for _, url := range urls {
@@ -724,7 +723,7 @@ func (i ActorResource) editActorExtRefs(req *restful.Request, resp *restful.Resp
 			}
 		}
 		if !found {
-			db.Delete(&link)
+			commonDb.Delete(&link)
 			models.AddActionActor(actor.ID, "edit_actor", "delete", "external_reference_link", link.ExternalReference.ExternalURL)
 		}
 	}

pkg/common/common.go

@@ -12,15 +12,16 @@ var (
 )
 
 type EnvConfigSpec struct {
-	Debug         bool   `envconfig:"DEBUG" default:"false"`
-	DebugRequests bool   `envconfig:"DEBUG_REQUESTS" default:"false"`
-	DebugSQL      bool   `envconfig:"DEBUG_SQL" default:"false"`
-	DebugWS       bool   `envconfig:"DEBUG_WS" default:"false"`
-	UIUsername    string `envconfig:"UI_USERNAME" required:"false"`
-	UIPassword    string `envconfig:"UI_PASSWORD" required:"false"`
-	DatabaseURL   string `envconfig:"DATABASE_URL" required:"false" default:""`
-	WsAddr        string `envconfig:"XBVR_WS_ADDR" required:"false" default:""`
-	WebPort       int    `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
+	Debug                bool   `envconfig:"DEBUG" default:"false"`
+	DebugRequests        bool   `envconfig:"DEBUG_REQUESTS" default:"false"`
+	DebugSQL             bool   `envconfig:"DEBUG_SQL" default:"false"`
+	DebugWS              bool   `envconfig:"DEBUG_WS" default:"false"`
+	UIUsername           string `envconfig:"UI_USERNAME" required:"false"`
+	UIPassword           string `envconfig:"UI_PASSWORD" required:"false"`
+	DatabaseURL          string `envconfig:"DATABASE_URL" required:"false" default:""`
+	WsAddr               string `envconfig:"XBVR_WS_ADDR" required:"false" default:""`
+	WebPort              int    `envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
+	DBConnectionPoolSize int    `envconfig:"DB_CONNECTION_POOL_SIZE" required:"false" default:"0"`
 }
 
 var EnvConfig EnvConfigSpec

pkg/common/paths.go

@@ -23,6 +23,7 @@ var ScriptHeatmapDir string
 var MyFilesDir string
 var DownloadDir string
 var WebPort int
+var DBConnectionPoolSize int
 
 func DirSize(path string) (int64, error) {
 	var size int64
@@ -51,6 +52,7 @@ func InitPaths() {
 	databaseurl := flag.String("database_url", "", "Optional: override default database path")
 	web_port := flag.Int("web_port", 0, "Optional: override default Web Page port 9999")
 	ws_addr := flag.String("ws_addr", "", "Optional: override default Websocket address from the default 0.0.0.0:9998")
+	db_connection_pool_size := flag.Int("db_connection_pool_size", 0, "Optional: sets a limit to the number of db connections while scraping")
 
 	flag.Parse()
 
@@ -113,6 +115,11 @@ func InitPaths() {
 			WsAddr = EnvConfig.WsAddr
 		}
 	}
+	if *db_connection_pool_size != 0 {
+		DBConnectionPoolSize = *db_connection_pool_size
+	} else {
+		DBConnectionPoolSize = EnvConfig.DBConnectionPoolSize
+	}
 
 	_ = os.MkdirAll(AppDir, os.ModePerm)
 	_ = os.MkdirAll(ImgDir, os.ModePerm)

pkg/models/db.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"strings"
+	"time"
 
 	"github.com/avast/retry-go/v4"
 	"github.com/jinzhu/gorm"
@@ -16,6 +17,7 @@ import (
 var log = &common.Log
 var dbConn *dburl.URL
 var supportedDB = []string{"mysql", "sqlite3"}
+var commonConnection *gorm.DB
 
 func parseDBConnString() {
 	var err error
@@ -78,6 +80,38 @@ func GetDB() (*gorm.DB, error) {
 	return db, nil
 }
 
+func GetCommonDB() (*gorm.DB, error) {
+	if common.EnvConfig.DebugSQL {
+		log.Debug("Getting Common DB handle from ", common.GetCallerFunctionName())
+	}
+
+	var err error
+
+	if commonConnection != nil {
+		return commonConnection, nil
+	}
+	err = retry.Do(
+		func() error {
+			commonConnection, err = gorm.Open(dbConn.Driver, dbConn.DSN)
+			commonConnection.LogMode(common.EnvConfig.DebugSQL)
+			commonConnection.DB().SetConnMaxIdleTime(4 * time.Minute)
+			if common.DBConnectionPoolSize > 0 {
+				commonConnection.DB().SetMaxOpenConns(common.DBConnectionPoolSize)
+			}
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	)
+
+	if err != nil {
+		log.Fatal("Failed to connect to database ", err)
+	}
+
+	return commonConnection, nil
+}
+
 // Lock functions
 
 func CreateLock(lock string) {
@@ -127,4 +161,5 @@ func init() {
 	common.InitPaths()
 	common.InitLogging()
 	parseDBConnString()
+	GetCommonDB()
 }

pkg/models/model_actor.go

@@ -97,12 +97,11 @@ type ActorLink struct {
 }
 
 func (i *Actor) Save() error {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
 	var err error = retry.Do(
 		func() error {
-			err := db.Save(&i).Error
+			err := commonDb.Save(&i).Error
 			if err != nil {
 				return err
 			}
@@ -118,8 +117,7 @@ func (i *Actor) Save() error {
 }
 
 func (i *Actor) CountActorTags() {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
 	type CountResults struct {
 		ID            int
@@ -131,7 +129,7 @@ func (i *Actor) CountActorTags() {
 
 	var results []CountResults
 
-	db.Model(&Actor{}).
+	commonDb.Model(&Actor{}).
 		Select("actors.id, count as existingcnt, count(*) cnt, sum(scenes.is_available ) is_available, avail_count as existingavail").
 		Group("actors.id").
 		Joins("join scene_cast on scene_cast.actor_id = actors.id").
@@ -141,7 +139,7 @@ func (i *Actor) CountActorTags() {
 	for i := range results {
 		var actor Actor
 		if results[i].Cnt != results[i].Existingcnt || results[i].IsAvailable != results[i].Existingavail {
-			db.First(&actor, results[i].ID)
+			commonDb.First(&actor, results[i].ID)
 			actor.Count = results[i].Cnt
 			actor.AvailCount = results[i].IsAvailable
 			actor.Save()
@@ -170,11 +168,10 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
 	limit := r.Limit.OrElse(100)
 	offset := r.Offset.OrElse(0)
 
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
 	var actors []Actor
-	tx := db.Model(&actors)
+	tx := commonDb.Model(&actors)
 
 	var out ResponseActorList
 
@@ -504,32 +501,29 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList {
 }
 
 func (o *Actor) GetIfExist(id string) error {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
-	return db.
+	return commonDb.
 		Preload("Scenes", func(db *gorm.DB) *gorm.DB {
 			return db.Where("is_hidden = 0")
 		}).
 		Where(&Actor{Name: id}).First(o).Error
 }
 
 func (o *Actor) GetIfExistByPK(id uint) error {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
-	return db.
+	return commonDb.
 		Preload("Scenes", func(db *gorm.DB) *gorm.DB {
 			return db.Where("is_hidden = 0")
 		}).
 		Where(&Actor{ID: id}).First(o).Error
 }
 
 func (o *Actor) GetIfExistByPKWithSceneAvg(id uint) error {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
-	tx := db.Model(&Actor{})
+	tx := commonDb.Model(&Actor{})
 	tx = tx.Select(`actors.*, 
 	(select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) as scene_rating_average`)
 
@@ -631,19 +625,17 @@ func addToStringArray(inputArray string, newValue string) (string, bool) {
 
 func (a *Actor) CheckForSetImage() bool {
 	// check if the field was deleted by the user,
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 	var action ActionActor
-	db.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action)
+	commonDb.Where("source = 'edit_actor' and actor_id = ? and changed_column = 'image_url' and action_type = 'setimage'", a.ID).Order("ID desc").First(&action)
 	return action.ID != 0
 }
 
 func (a *Actor) CheckForUserDeletes(fieldName string, newValue string) bool {
 	// check if the field was deleted by the user,
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 	var action ActionActor
-	db.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action)
+	commonDb.Where("source = 'edit_actor' and actor_id = ? and changed_column = ? and new_value = ?", a.ID, fieldName, newValue).Order("ID desc").First(&action)
 	if action.ID != 0 && action.ActionType == "delete" {
 		return true
 	}

pkg/models/model_aka.go

@@ -20,12 +20,11 @@ type Aka struct {
 }
 
 func (i *Aka) Save() error {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
 	err := retry.Do(
 		func() error {
-			err := db.Save(&i).Error
+			err := commonDb.Save(&i).Error
 			if err != nil {
 				return err
 			}
@@ -41,24 +40,22 @@ func (i *Aka) Save() error {
 }
 
 func (o *Aka) GetIfExistByPK(id uint) error {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
-	return db.
+	return commonDb.
 		Preload("Actors").
 		Where(&Aka{ID: id}).First(o).Error
 }
 
 func (o *Aka) UpdateAkaSceneCastRecords() {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
 	// Queries to update the scene_cast table for the aka actor are comlex but fast.
 	//  Significating faster than iterating through the results of multiple simpler queries.
 	// 	The Raw Sql used is compatible between mysql & sqlite
 
 	// add missing scene_cast records for aka actors
-	db.Exec(`
+	commonDb.Exec(`
 	insert into scene_cast 
 	select distinct sc.scene_id, a.aka_actor_id 
 	from akas a 
@@ -69,7 +66,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() {
 	`)
 
 	// delete scene_cast records for aka actors that have been removed
-	db.Exec(`
+	commonDb.Exec(`
 		with SceneIds as (
 			select distinct a.id, sc.scene_id
 			from akas a 
@@ -95,8 +92,7 @@ func (o *Aka) UpdateAkaSceneCastRecords() {
 }
 
 func (o *Aka) RefreshAkaActorNames() {
-	db, _ := GetDB()
-	defer db.Close()
+	commonDb, _ := GetCommonDB()
 
 	type SortedList struct {
 		AkaActorId uint
@@ -105,7 +101,7 @@ func (o *Aka) RefreshAkaActorNames() {
 	var sortedList []SortedList
 
 	// this update the aka names by reordering the actor names based on descending count
-	db.Raw(`
+	commonDb.Raw(`
 	with sorted as (
 		select a.aka_actor_id, a2.name, a2.count  from akas a 
 		join actor_akas aa on a.id =aa.aka_id 
@@ -118,7 +114,7 @@ func (o *Aka) RefreshAkaActorNames() {
 	for _, listItem := range sortedList {
 		var actor Actor
 		actor.ID = listItem.AkaActorId
-		db.Model(&actor).Where("name != ?", "aka:"+listItem.SortedName).Update("name", "aka:"+listItem.SortedName)
+		commonDb.Model(&actor).Where("name != ?", "aka:"+listItem.SortedName).Update("name", "aka:"+listItem.SortedName)
 	}
 }