Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Profile performance improvements #1314

Merged
merged 3 commits into from
Oct 17, 2024
Merged

Profile performance improvements #1314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
06b5987
incusd/db/profiles: Support config caching
stgraber Oct 17, 2024
8f46ded
incusd: Update calls to profile ToAPI
stgraber Oct 17, 2024
8261a78
incusd/cluster/profiles: Fix import shadowing
stgraber Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cmd/incusd/api_internal_recover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ func internalRecoverScan(ctx context.Context, s *state.State, userPools []api.St
return err
}

profileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

profileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
Expand All @@ -92,7 +97,7 @@ func internalRecoverScan(ctx context.Context, s *state.State, userPools []api.St
projectProfiles[profile.Project] = []*api.Profile{}
}

apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/incusd/instance_patch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,13 +192,18 @@ func instancePatch(d *Daemon, r *http.Request) response.Response {
return err
}

profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

for _, profile := range profiles {
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/incusd/instance_post.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -636,13 +636,18 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance
return err
}

profileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

profileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

for _, profile := range rawProfiles {
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/incusd/instance_put.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,18 @@ func instancePut(d *Daemon, r *http.Request) response.Response {
return err
}

profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

for _, profile := range profiles {
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down
7 changes: 6 additions & 1 deletion cmd/incusd/instances_post.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -993,6 +993,11 @@ func instancesPost(d *Daemon, r *http.Request) response.Response {
return err
}

dbProfileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

dbProfileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
Expand All @@ -1009,7 +1014,7 @@ func instancesPost(d *Daemon, r *http.Request) response.Response {
return fmt.Errorf("Requested profile %q doesn't exist", profileName)
}

apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileConfigs, dbProfileDevices)
if err != nil {
return err
}
Expand Down
18 changes: 14 additions & 4 deletions cmd/incusd/profiles.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,11 @@ func profilesGet(d *Daemon, r *http.Request) response.Response {
}

if recursion {
profileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

profileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
Expand All @@ -201,7 +206,7 @@ func profilesGet(d *Daemon, r *http.Request) response.Response {
continue
}

apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down Expand Up @@ -437,12 +442,17 @@ func profileGet(d *Daemon, r *http.Request) response.Response {
return fmt.Errorf("Fetch profile: %w", err)
}

profileConfigs, err := dbCluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

profileDevices, err := dbCluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

resp, err = profile.ToAPI(ctx, tx.Tx(), profileDevices)
resp, err = profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down Expand Up @@ -533,7 +543,7 @@ func profilePut(d *Daemon, r *http.Request) response.Response {
return fmt.Errorf("Failed to retrieve profile %q: %w", name, err)
}

profile, err = current.ToAPI(ctx, tx.Tx(), nil)
profile, err = current.ToAPI(ctx, tx.Tx(), nil, nil)
if err != nil {
return err
}
Expand Down Expand Up @@ -638,7 +648,7 @@ func profilePatch(d *Daemon, r *http.Request) response.Response {
return fmt.Errorf("Failed to retrieve profile=%q: %w", name, err)
}

profile, err = current.ToAPI(ctx, tx.Tx(), nil)
profile, err = current.ToAPI(ctx, tx.Tx(), nil, nil)
if err != nil {
return err
}
Expand Down
8 changes: 7 additions & 1 deletion internal/server/backup/backup_config_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,20 @@ func ConfigToInstanceDBArgs(state *state.State, c *config.Config, projectName st
return err
}

// Get all the profile configs.
profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

// Get all the profile devices.
profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return err
}

for _, profile := range profiles {
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down
11 changes: 9 additions & 2 deletions internal/server/db/cluster/instances.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,19 @@ type InstanceFilter struct {
}

// ToAPI converts the database Instance to API type.
func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, instanceDevices map[int][]Device, profileDevices map[int][]Device) (*api.Instance, error) {
func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, instanceDevices map[int][]Device, profileConfigs map[int]map[string]string, profileDevices map[int][]Device) (*api.Instance, error) {
profiles, err := GetInstanceProfiles(ctx, tx, i.ID)
if err != nil {
return nil, err
}

if profileConfigs == nil {
profileConfigs, err = GetConfig(ctx, tx, "profile")
if err != nil {
return nil, err
}
}

if profileDevices == nil {
profileDevices, err = GetDevices(ctx, tx, "profile")
if err != nil {
Expand All @@ -93,7 +100,7 @@ func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, instanceDevices map[in
apiProfiles := make([]api.Profile, 0, len(profiles))
profileNames := make([]string, 0, len(profiles))
for _, p := range profiles {
apiProfile, err := p.ToAPI(ctx, tx, profileDevices)
apiProfile, err := p.ToAPI(ctx, tx, profileConfigs, profileDevices)
if err != nil {
return nil, err
}
Expand Down
30 changes: 20 additions & 10 deletions internal/server/db/cluster/profiles.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,21 +52,31 @@ type ProfileFilter struct {
}

// ToAPI returns a cluster Profile as an API struct.
func (p *Profile) ToAPI(ctx context.Context, tx *sql.Tx, profileDevices map[int][]Device) (*api.Profile, error) {
config, err := GetProfileConfig(ctx, tx, p.ID)
if err != nil {
return nil, err
func (p *Profile) ToAPI(ctx context.Context, tx *sql.Tx, profileConfigs map[int]map[string]string, profileDevices map[int][]Device) (*api.Profile, error) {
var err error

var dbConfig map[string]string
if profileConfigs != nil {
dbConfig = profileConfigs[p.ID]
if dbConfig == nil {
dbConfig = map[string]string{}
}
} else {
dbConfig, err = GetProfileConfig(ctx, tx, p.ID)
if err != nil {
return nil, err
}
}

var devices map[string]Device
var dbDevices map[string]Device
if profileDevices != nil {
devices = map[string]Device{}
dbDevices = map[string]Device{}

for _, dev := range profileDevices[p.ID] {
devices[dev.Name] = dev
dbDevices[dev.Name] = dev
}
} else {
devices, err = GetProfileDevices(ctx, tx, p.ID)
dbDevices, err = GetProfileDevices(ctx, tx, p.ID)
if err != nil {
return nil, err
}
Expand All @@ -76,8 +86,8 @@ func (p *Profile) ToAPI(ctx context.Context, tx *sql.Tx, profileDevices map[int]
Name: p.Name,
ProfilePut: api.ProfilePut{
Description: p.Description,
Config: config,
Devices: DevicesToAPI(devices),
Config: dbConfig,
Devices: DevicesToAPI(dbDevices),
},
Project: p.Project,
}
Expand Down
8 changes: 7 additions & 1 deletion internal/server/db/instances.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,12 @@ func (c *ClusterTx) instanceProfilesFill(ctx context.Context, snapshotsMode bool
return fmt.Errorf("Failed loading profiles: %w", err)
}

// Get all the profile configs.
profileConfigs, err := cluster.GetConfig(context.TODO(), c.Tx(), "profile")
if err != nil {
return fmt.Errorf("Failed loading profile configs: %w", err)
}

// Get all the profile devices.
profileDevices, err := cluster.GetDevices(context.TODO(), c.Tx(), "profile")
if err != nil {
Expand All @@ -549,7 +555,7 @@ func (c *ClusterTx) instanceProfilesFill(ctx context.Context, snapshotsMode bool
continue
}

profilesByID[profile.ID], err = profile.ToAPI(context.TODO(), c.tx, profileDevices)
profilesByID[profile.ID], err = profile.ToAPI(context.TODO(), c.tx, profileConfigs, profileDevices)
if err != nil {
return err
}
Expand Down
10 changes: 8 additions & 2 deletions internal/server/db/profiles.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (c *ClusterTx) GetProfile(ctx context.Context, project, name string) (int64
profile := profiles[0]
id := int64(profile.ID)

result, err := profile.ToAPI(ctx, c.tx, nil)
result, err := profile.ToAPI(ctx, c.tx, nil, nil)
if err != nil {
return -1, nil, err
}
Expand All @@ -78,14 +78,20 @@ func (c *ClusterTx) GetProfiles(ctx context.Context, projectName string, profile
return nil, err
}

// Get all the profile configs.
profileConfigs, err := cluster.GetConfig(ctx, c.Tx(), "profile")
if err != nil {
return nil, err
}

// Get all the profile devices.
profileDevices, err := cluster.GetDevices(ctx, c.Tx(), "profile")
if err != nil {
return nil, err
}

for i, profile := range dbProfiles {
apiProfile, err := profile.ToAPI(ctx, c.tx, profileDevices)
apiProfile, err := profile.ToAPI(ctx, c.tx, profileConfigs, profileDevices)
if err != nil {
return nil, err
}
Expand Down
9 changes: 7 additions & 2 deletions internal/server/project/permissions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1257,14 +1257,19 @@ func fetchProject(tx *db.ClusterTx, projectName string, skipIfNoLimits bool) (*p
return nil, fmt.Errorf("Fetch profiles from database: %w", err)
}

dbProfileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return nil, fmt.Errorf("Fetch profile configs from database: %w", err)
}

dbProfileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return nil, fmt.Errorf("Fetch profile devices from database: %w", err)
}

profiles := make([]api.Profile, 0, len(dbProfiles))
for _, profile := range dbProfiles {
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileConfigs, dbProfileDevices)
if err != nil {
return nil, err
}
Expand All @@ -1284,7 +1289,7 @@ func fetchProject(tx *db.ClusterTx, projectName string, skipIfNoLimits bool) (*p

instances := make([]api.Instance, 0, len(dbInstances))
for _, instance := range dbInstances {
apiInstance, err := instance.ToAPI(ctx, tx.Tx(), dbInstanceDevices, dbProfileDevices)
apiInstance, err := instance.ToAPI(ctx, tx.Tx(), dbInstanceDevices, dbProfileConfigs, dbProfileDevices)
if err != nil {
return nil, fmt.Errorf("Failed to get API data for instance %q in project %q: %w", instance.Name, instance.Project, err)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/server/scriptlet/instance_placement.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ func InstancePlacementRun(ctx context.Context, l logger.Logger, s *state.State,

// Convert the []Instances into []api.Instances.
for _, obj := range objects {
instance, err := obj.ToAPI(ctx, tx.Tx(), objectDevices, nil)
instance, err := obj.ToAPI(ctx, tx.Tx(), objectDevices, nil, nil)
if err != nil {
return err
}
Expand Down
10 changes: 8 additions & 2 deletions internal/server/storage/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,14 +821,20 @@ func VolumeUsedByProfileDevices(s *state.State, poolName string, projectName str
return fmt.Errorf("Failed loading profiles: %w", err)
}

// Get all the profile configs.
profileConfigs, err := cluster.GetConfig(ctx, tx.Tx(), "profile")
if err != nil {
return fmt.Errorf("Failed loading profile configs: %w", err)
}

// Get all the profile devices.
profileDevices, err := cluster.GetDevices(ctx, tx.Tx(), "profile")
if err != nil {
return fmt.Errorf("Failed loading profiles: %w", err)
return fmt.Errorf("Failed loading profile devices: %w", err)
}

for _, profile := range dbProfiles {
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileDevices)
apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices)
if err != nil {
return fmt.Errorf("Failed getting API Profile %q: %w", profile.Name, err)
}
Expand Down
Loading