feature: dameng database support (#19570)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

pkg/cloudcommon/database.go

@@ -17,6 +17,7 @@ package cloudcommon
 import (
 	"context"
 	"database/sql"
+	"strings"
 	"time"
 
 	"github.com/mattn/go-sqlite3"
@@ -25,6 +26,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
+	_ "yunion.io/x/sqlchemy/backends"
 
 	noapi "yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
@@ -43,6 +45,8 @@ func InitDB(options *common_options.DBOptions) {
 		sqlchemy.DEBUG_SQLCHEMY = true
 	}
 
+	log.Infof("Registered SQL drivers: %s", strings.Join(sql.Drivers(), ", "))
+
 	consts.QueryOffsetOptimization = options.QueryOffsetOptimization
 
 	if options.HistoricalUniqueName {
@@ -64,6 +68,10 @@ func InitDB(options *common_options.DBOptions) {
 	}
 	backend := sqlchemy.MySQLBackend
 	switch dialect {
+	case "dm":
+		backend = sqlchemy.DamengBackend
+		dialect = "dm"
+		sqlStr = "dm://" + sqlStr
 	case "sqlite3":
 		backend = sqlchemy.SQLiteBackend
 		dialect = "sqlite3_with_extensions"

pkg/cloudcommon/db/modelbase.go

@@ -580,7 +580,7 @@ func (manager *SModelBaseManager) CustomizedTotalCount(ctx context.Context, user
 	ret := apis.TotalCountBase{}
 	err := totalQ.First(&ret)
 	if err != nil {
-		return -1, nil, errors.Wrap(err, "SModelBaseManager Query total")
+		return -1, nil, errors.Wrapf(err, "SModelBaseManager Query total %s", totalQ.DebugString())
 	}
 	return ret.Count, nil, nil
 }

pkg/cloudcommon/db/tablespec.go

@@ -17,6 +17,7 @@ package db
 import (
 	"context"
 	"reflect"
+	"runtime/debug"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -277,6 +278,7 @@ func (ts *sTableSpec) inform(ctx context.Context, dt interface{}, f func(ctx con
 		obj, err := ts.newInformerModel(dt)
 		if err != nil {
 			log.Warningf("newInformerModel error: %v", err)
+			debug.PrintStack()
 			return
 		}
 		if err := f(ctx, obj); err != nil {
@@ -298,6 +300,7 @@ func (ts *sTableSpec) informUpdate(ctx context.Context, dt interface{}, oldObj *
 		obj, err := ts.newInformerModel(dt)
 		if err != nil {
 			log.Warningf("newInformerModel error: %v", err)
+			debug.PrintStack()
 			return
 		}
 		if err := informer.Update(ctx, obj, oldObj); err != nil {

pkg/compute/models/networkaddresses.go

@@ -517,12 +517,15 @@ func (man *SNetworkAddressManager) ListItemFilter(ctx context.Context, q *sqlche
 	q, err = managedResourceFilterByAccount(
 		ctx,
 		q, input.ManagedResourceListInput, "network_id", func() *sqlchemy.SQuery {
-			networks := NetworkManager.Query().SubQuery()
-			wires := WireManager.Query().SubQuery()
-			vpcs := VpcManager.Query().SubQuery()
-			subq := networks.Query(networks.Field("id"))
-			subq = subq.Join(wires, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
-			subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+			networks := NetworkManager.Query()
+			wires := WireManager.Query("id", "vpc_id", "manager_id").SubQuery()
+			vpcs := VpcManager.Query("id", "manager_id").SubQuery()
+			networks = networks.Join(wires, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
+			networks = networks.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+			networks = networks.AppendField(networks.Field("id"))
+			networks = networks.AppendField(sqlchemy.NewFunction(sqlchemy.NewCase().When(sqlchemy.IsNullOrEmpty(wires.Field("manager_id")), vpcs.Field("manager_id")).Else(wires.Field("manager_id")), "manager_id"))
+			subq := networks.SubQuery().Query()
+			subq = subq.AppendField(subq.Field("id"))
 			return subq
 		})
 	if err != nil {
@@ -537,13 +540,17 @@ func (man *SNetworkAddressManager) OrderByExtraFields(
 	q *sqlchemy.SQuery,
 	userCred mcclient.TokenCredential,
 	query api.NetworkAddressListInput,
-) (retq *sqlchemy.SQuery, err error) {
-	retq, err = man.SNetworkResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.NetworkFilterListInput)
+) (*sqlchemy.SQuery, error) {
+	var err error
+	q, err = man.SNetworkResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.NetworkFilterListInput)
+	if err != nil {
+		return nil, err
+	}
+	q, err = man.SStandaloneAnonResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneAnonResourceListInput)
 	if err != nil {
 		return nil, err
 	}
-	retq, err = man.SStandaloneAnonResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneAnonResourceListInput)
-	return retq, nil
+	return q, nil
 }
 
 func (man *SNetworkAddressManager) FetchCustomizeColumns(

pkg/compute/models/server_skus.go

@@ -98,19 +98,19 @@ type SServerSku struct {
 	SysDiskMinSizeGB int               `nullable:"true" list:"user" create:"admin_optional" update:"admin"` // not required。 windows比较新的版本都是50G左右。
 	SysDiskMaxSizeGB int               `nullable:"true" list:"user" create:"admin_optional" update:"admin"` // not required
 
-	AttachedDiskType   string `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
+	AttachedDiskType   string `width:"32" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 	AttachedDiskSizeGB int    `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 	AttachedDiskCount  int    `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 
 	DataDiskTypes    string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 	DataDiskMaxCount int    `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 
-	NicType     string `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
+	NicType     string `width:"32" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 	NicMaxCount int    `default:"1" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 
 	GpuAttachable tristate.TriState `default:"true" list:"user" create:"admin_optional" update:"admin"`
 	GpuSpec       string            `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
-	GpuCount      string            `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
+	GpuCount      string            `width:"16" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 	GpuMaxCount   int               `nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 
 	Provider string `width:"64" charset:"ascii" nullable:"true" list:"user" default:"OneCloud" create:"admin_optional"`
@@ -556,7 +556,7 @@ func (manager *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context,
 	q = q.Asc(q.Field("cpu_core_count"), q.Field("memory_size_mb"))
 	err = db.FetchModelObjects(manager, q, &skus)
 	if err != nil {
-		log.Infof("FetchModelObjects %s", err)
+		log.Errorf("FetchModelObjects %s: %s", q.DebugString(), err)
 		return nil, httperrors.NewBadRequestError("instance specs list query error")
 	}
 

pkg/keystone/models/tokencache.go

@@ -158,7 +158,7 @@ func (manager *STokenCacheManager) FetchToken(tokenStr string) (*STokenCache, er
 }
 
 func (manager *STokenCacheManager) removeObsolete() error {
-	sql := fmt.Sprintf("DELETE FROM `%s` WHERE `expired_at` < ?", manager.TableSpec().Name())
+	sql := fmt.Sprintf("DELETE FROM %s WHERE expired_at < ?", manager.TableSpec().Name())
 	db := sqlchemy.GetDBWithName(manager.TableSpec().GetDBName())
 	now := timeutils.UtcNow()
 	_, err := db.Exec(sql, now.Add(-24*time.Hour))

pkg/util/splitable/sync.go

@@ -133,8 +133,13 @@ func (spec *SSplitTableSpec) SyncSQL() []string {
 			Start: indexCol.AutoIncrementOffset(),
 		}
 		// insert the first meta
-		sql := fmt.Sprintf("INSERT INTO `%s`(`table`, `deleted`, `created_at`) VALUES('%s', 0, '%s')", spec.metaSpec.Name(), meta.Table, timeutils.MysqlTime(now))
-		sqls = append(sqls, sql)
+		insertResult, err := spec.metaSpec.InsertSqlPrep(&meta, false)
+		if err != nil {
+			log.Errorf("spec.metaSpec.InsertSqlPrep fail %s", err)
+			return nil
+		}
+		// sql := fmt.Sprintf("INSERT INTO `%s`(`table`, `deleted`, `created_at`) VALUES('%s', 0, '%s')", spec.metaSpec.Name(), meta.Table, timeutils.MysqlTime(now))
+		sqls = append(sqls, sqlchemy.SQLPrintf(insertResult.Sql, insertResult.Values))
 		// create the first table
 		newtable := spec.GetTableSpec(meta)
 		nsql := newtable.SyncSQL()