Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 2523 ce #2524

Merged
merged 4 commits into from
Aug 6, 2024
Merged

Issue 2523 ce #2524

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3510f91
fix: sync audit plan scheduled task failure
iwanghc Aug 6, 2024
bebb33d
fix: mysql slowlog audit plan lose start time
iwanghc Aug 6, 2024
f5f7e16
modify：get latest audit plan records sql modify
iwanghc Aug 6, 2024
cf02a5f
modify: modify audit plan metric name
iwanghc Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions sqle/model/instance_audit_plan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,11 @@ func (s *Storage) UpdateInstanceAuditPlanByID(id uint, attrs map[string]interfac
}

// GetLatestAuditPlanIds 获取所有变更过的记录，包括删除
func (s *Storage) GetLatestAuditPlanRecordsV2(after time.Time) ([]*AuditPlanDetail, error) {
// 采集时会更新last_collection_time会同步更新updated_at，此处获取updated_at > last_collection_time的任务，即为配置变更过的任务
// 影响：会查出所有被删除的任务，在syncTask时做一次额外的删除操作
func (s *Storage) GetLatestAuditPlanRecordsV2() ([]*AuditPlanDetail, error) {
var aps []*AuditPlanDetail
err := s.db.Unscoped().Model(AuditPlanV2{}).Select("audit_plans_v2.id, audit_plans_v2.updated_at").Where("audit_plans_v2.updated_at > ?", after).Order("updated_at").Scan(&aps).Error
err := s.db.Unscoped().Model(AuditPlanV2{}).Select("audit_plans_v2.id, audit_plans_v2.updated_at,audit_plans_v2.last_collection_time").Where("audit_plans_v2.updated_at > audit_plans_v2.last_collection_time").Order("updated_at").Scan(&aps).Error
return aps, errors.New(errors.ConnectStorageError, err)
}

Expand All @@ -117,7 +119,7 @@ type AuditPlanV2 struct {
RuleTemplateName string `json:"rule_template_name" gorm:"type:varchar(255)"`
Params params.Params `json:"params" gorm:"type:varchar(1000)"`
ActiveStatus string `json:"active_status" gorm:"type:varchar(255)"`
LastCollectionTime *time.Time `json:"last_collection_time" gorm:"type:datetime"`
LastCollectionTime *time.Time `json:"last_collection_time" gorm:"type:datetime(3)"`

AuditPlanSQLs []*SQLManageRecord `gorm:"foreignKey:SourceId"`
}
Expand Down Expand Up @@ -151,7 +153,7 @@ func (s *Storage) GetLatestStartTimeAuditPlanSQLV2(sourceId uint) (string, error
var info = struct {
StartTime string `gorm:"column:max_start_time"`
}{}
err := s.db.Raw(`SELECT MAX(STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(info, '$.start_time')), '%Y-%m-%dT%H:%i:%s.%fZ'))
err := s.db.Raw(`SELECT MAX(STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(info, '$.start_time')), '%Y-%m-%dT%H:%i:%s.%f'))
AS max_start_time FROM sql_manage_records WHERE source_id = ?`, sourceId).Scan(&info).Error
return info.StartTime, err
}
Expand Down Expand Up @@ -415,5 +417,6 @@ func (s *Storage) UpdateManagerSQLStatus(sql *SQLManageRecord) error {
}

func (s *Storage) UpdateAuditPlanLastCollectionTime(auditPlanID uint, collectionTime time.Time) error {
return s.db.Model(AuditPlanV2{}).Where("id = ?", auditPlanID).Update("last_collection_time", collectionTime).Error
const query = `UPDATE audit_plans_v2 SET last_collection_time = now(3) WHERE id = ?;`
return s.db.Exec(query, auditPlanID).Error
}
3 changes: 1 addition & 2 deletions sqle/server/auditplan/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func (mgr *Manager) sync() error {
}
}
// 增量同步智能扫描任务，根据数据库记录的更新时间筛选，更新后将下次筛选的时间为上一次记录的最晚的更新时间。
aps, err := mgr.persist.GetLatestAuditPlanRecordsV2(*mgr.lastSyncTime)
aps, err := mgr.persist.GetLatestAuditPlanRecordsV2()
if err != nil {
return err
}
Expand All @@ -239,7 +239,6 @@ func (mgr *Manager) sync() error {
if err != nil {
mgr.logger.WithField("id", ap.ID).Errorf("sync audit task failed, error: %v", err)
}
mgr.lastSyncTime = &ap.UpdatedAt
}
return nil
}
Expand Down
24 changes: 13 additions & 11 deletions sqle/server/auditplan/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const MetricNameRowExaminedAvg string = "row_examined_avg" // 平均扫描行数
const MetricNameFirstQueryAt string = "first_query_at"
const MetricNameDBUser string = "db_user"
const MetricNameEndpoints string = "endpoints"
const MetricNameStartTimeOfLastScrapedSQL string = "start_time_of_last_scraped_sql" // 抓取sql的开始时间

const MetricNameMetaName string = "schema_meta_name" // 表或者视图的名字
const MetricNameMetaType string = "schema_meta_type" // 表或者视图等等
Expand All @@ -35,17 +36,18 @@ const MetricNameDiskReadAvg = "disk_read_avg"
const MetricNameBufferReadAvg = "buffer_read_avg"

var ALLMetric = map[string]MetricType{
MetricNameCounter: MetricTypeInt, // MySQL slow log
MetricNameLastReceiveTimestamp: MetricTypeString, // MySQL slow log
MetricNameQueryTimeAvg: MetricTypeFloat, // MySQL slow log
MetricNameQueryTimeMax: MetricTypeFloat, // MySQL slow log
MetricNameRowExaminedAvg: MetricTypeFloat, // MySQL slow log
MetricNameFirstQueryAt: MetricTypeString, // MySQL slow log, 好像没用上 | OB MySQL TOP SQL
MetricNameDBUser: MetricTypeString, // MySQL slow log
MetricNameEndpoints: MetricTypeString, // MySQL slow log
MetricNameMetaName: MetricTypeString, // MySQL schema meta
MetricNameMetaType: MetricTypeString, // MySQL schema meta
MetricNameRecordDeleted: MetricTypeBool, // MySQL schema meta
MetricNameCounter: MetricTypeInt, // MySQL slow log
MetricNameLastReceiveTimestamp: MetricTypeString, // MySQL slow log
MetricNameQueryTimeAvg: MetricTypeFloat, // MySQL slow log
MetricNameQueryTimeMax: MetricTypeFloat, // MySQL slow log
MetricNameRowExaminedAvg: MetricTypeFloat, // MySQL slow log
MetricNameFirstQueryAt: MetricTypeString, // MySQL slow log, 好像没用上 | OB MySQL TOP SQL
MetricNameDBUser: MetricTypeString, // MySQL slow log
MetricNameEndpoints: MetricTypeString, // MySQL slow log
MetricNameStartTimeOfLastScrapedSQL: MetricTypeString, // MySQL slow log
MetricNameMetaName: MetricTypeString, // MySQL schema meta
MetricNameMetaType: MetricTypeString, // MySQL schema meta
MetricNameRecordDeleted: MetricTypeBool, // MySQL schema meta

MetricNameQueryTimeTotal: MetricTypeInt, // DB2 TOP SQL | OB Oracle TOP SQL
MetricNameCPUTimeAvg: MetricTypeFloat, // DB2 TOP SQL | OB MySQL TOP SQL
Expand Down
Loading