Enable MC-Infra dynamic provisioning with custom image

go.mod

@@ -89,7 +89,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.26.0
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect

go.sum

@@ -13,14 +13,6 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0
 github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
 github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
 github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cloud-barista/mc-terrarium v0.0.10 h1:WrK/bHyiQZ9ZxRLVoUTY4c3CVH7z+vEzdh7yxo9X4ZM=
-github.com/cloud-barista/mc-terrarium v0.0.10/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I=
-github.com/cloud-barista/mc-terrarium v0.0.15 h1:1yJjCsNMwYZJyYSku9HQPOdj1PwAf5ybYI+DqB1TKsc=
-github.com/cloud-barista/mc-terrarium v0.0.15/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I=
-github.com/cloud-barista/mc-terrarium v0.0.16 h1:0TpSnFk5IpStpQc1YJSbMzB8q1jgVZ639Cjh7YAyxh4=
-github.com/cloud-barista/mc-terrarium v0.0.16/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I=
-github.com/cloud-barista/mc-terrarium v0.0.17 h1:W6kyGc2dIKrOMnw0UAOoI66G/bX9Tj/wG4pY9TN9uqI=
-github.com/cloud-barista/mc-terrarium v0.0.17/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I=
 github.com/cloud-barista/mc-terrarium v0.0.18 h1:uDzFOIOoIb7hSXJ5ovrrx8+/hqjHeyNNMU3EDfv8If0=
 github.com/cloud-barista/mc-terrarium v0.0.18/go.mod h1:iQxZNRa04d7mHA0h5dEPfF7ch1SBUS/ZFGUynKsKJ6I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=

src/api/rest/server/server.go

@@ -101,6 +101,11 @@ func RunServer() {
 	APILogSkipPatterns := [][]string{
 		{"/tumblebug/api"},
 		{"/mci", "option=status"},
+		{"/k8scluster"},
+		{"/resources/vNet"},
+		{"/resources/securityGroup"},
+		{"/resources/vpn"},
+		{"/resources/sshKey"},
 	}
 	e.Use(middlewares.Zerologger(APILogSkipPatterns))
 

src/core/common/label/label.go

@@ -250,7 +250,7 @@ func GetResourcesByLabelSelector(labelType, labelSelector string) ([]interface{}
 	keyValue = kvutil.FilterKvListBy(keyValue, listKey, 1)
 
 	// Log the number of filtered label entries
-	log.Info().Int("numLabelEntries", len(keyValue)).Str("listKey", listKey).Msg("Fetched and filtered list of label entries")
+	//log.Debug().Int("numLabelEntries", len(keyValue)).Str("listKey", listKey).Msg("Fetched and filtered list of label entries")
 
 	// Get the appropriate resource type constructor
 	resourceConstructor, exists := model.ResourceTypeRegistry[labelType]
@@ -264,7 +264,7 @@ func GetResourcesByLabelSelector(labelType, labelSelector string) ([]interface{}
 		labelKey := kv.Key
 		labelData := kv.Value
 
-		log.Info().Str("labelKey", labelKey).Msg("Processing label entry")
+		log.Debug().Str("labelKey", labelKey).Msg("Processing label entry")
 		//log.Debug().Str("labelKey", labelKey).Str("labelData", string(labelData)).Msg("Fetched label data")
 
 		var labelInfo model.LabelInfo
@@ -301,6 +301,6 @@ func GetResourcesByLabelSelector(labelType, labelSelector string) ([]interface{}
 		}
 	}
 
-	log.Info().Int("numMatchedResources", len(matchedResources)).Str("labelType", labelType).Msg("Matched resources found")
+	//log.Debug().Int("numMatchedResources", len(matchedResources)).Str("labelType", labelType).Msg("Matched resources found")
 	return matchedResources, nil
 }

src/core/infra/provisioning.go

@@ -946,7 +946,7 @@ func CreateMciDynamic(reqID string, nsId string, req *model.TbMciDynamicReq, dep
 	// Check whether VM names meet requirement.
 	errStr := ""
 	for i, k := range vmRequest {
-		err = checkCommonResAvailableForVmDynamicReq(&k)
+		err = checkCommonResAvailableForVmDynamicReq(&k, nsId)
 		if err != nil {
 			log.Error().Err(err).Msgf("[%d] Failed to find common resource for MCI provision", i)
 			errStr += "{[" + strconv.Itoa(i+1) + "] " + err.Error() + "} "
@@ -1046,7 +1046,7 @@ func CreateMciVmDynamic(nsId string, mciId string, req *model.TbVmDynamicReq) (*
 }
 
 // checkCommonResAvailableForVmDynamicReq is func to check common resources availability for VmDynamicReq
-func checkCommonResAvailableForVmDynamicReq(req *model.TbVmDynamicReq) error {
+func checkCommonResAvailableForVmDynamicReq(req *model.TbVmDynamicReq, nsId string) error {
 
 	vmRequest := req
 	// Check whether VM names meet requirement.
@@ -1076,20 +1076,20 @@ func checkCommonResAvailableForVmDynamicReq(req *model.TbVmDynamicReq) error {
 		return err
 	}
 
-	osType := strings.ReplaceAll(k.CommonImage, " ", "")
-	vmReq.ImageId = resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", osType)
-	// incase of user provided image id completely (e.g. aws+ap-northeast-2+ubuntu22.04)
-	if strings.Contains(k.CommonImage, "+") {
-		vmReq.ImageId = k.CommonImage
+	// 1) try if there is matched custom image in the namespace
+	_, err = resource.GetImage(nsId, k.CommonImage)
+	if err == nil {
+		return nil
 	}
-	_, err = resource.GetImage(model.SystemCommonNs, vmReq.ImageId)
-	if err != nil {
-		err := fmt.Errorf("Failed to get Image " + k.CommonImage + " from " + vmReq.ConnectionName)
-		log.Error().Err(err).Msg("")
-		return err
+
+	// 2) try if there is a matched image with modified ImageId by osType in the CommonImage list
+	modifiedImageKey := resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", k.CommonImage)
+	_, err = resource.GetImage(model.SystemCommonNs, modifiedImageKey)
+	if err == nil {
+		return nil
 	}
 
-	return nil
+	return err
 }
 
 // getVmReqForDynamicMci is func to getVmReqFromDynamicReq
@@ -1129,17 +1129,15 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *model.TbVmDynamicReq
 	resourceName := nsId + model.StrSharedResourceName + vmReq.ConnectionName
 
 	vmReq.SpecId = specInfo.Id
-	osType := strings.ReplaceAll(k.CommonImage, " ", "")
-	vmReq.ImageId = resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", osType)
-	// incase of user provided image id completely (e.g. aws+ap-northeast-2+ubuntu22.04)
-	if strings.Contains(k.CommonImage, "+") {
-		vmReq.ImageId = k.CommonImage
-	}
-	_, err = resource.GetImage(model.SystemCommonNs, vmReq.ImageId)
+	vmReq.ImageId = k.CommonImage
+	_, err = resource.GetImage(nsId, k.CommonImage)
 	if err != nil {
-		err := fmt.Errorf("Failed to get the Image " + vmReq.ImageId + " from " + vmReq.ConnectionName)
-		log.Error().Err(err).Msg("")
-		return &model.TbVmReq{}, err
+		// set modifiedImageKey if there is no matched custom image in the namespace
+		modifiedImageKey := resource.GetProviderRegionZoneResourceKey(connection.ProviderName, connection.RegionDetail.RegionName, "", k.CommonImage)
+		_, err = resource.GetImage(model.SystemCommonNs, modifiedImageKey)
+		if err == nil {
+			vmReq.ImageId = modifiedImageKey
+		}
 	}
 
 	/*
@@ -1157,7 +1155,11 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *model.TbVmDynamicReq
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to create etcd session")
 	}
-	defer session.Close()
+	defer func() {
+		if err := session.Close(); err != nil {
+			log.Error().Err(err).Msg("Failed to close etcd session")
+		}
+	}()
 
 	lock, err := kvstore.NewLock(ctx, session, vNetKey)
 	if err != nil {
@@ -1168,8 +1170,12 @@ func getVmReqFromDynamicReq(reqID string, nsId string, req *model.TbVmDynamicReq
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to acquire lock")
 	}
-	// Unlock the lock when the function exits
-	defer lock.Unlock(ctx)
+	// Ensure the lock is released when the function exits
+	defer func() {
+		if err := lock.Unlock(ctx); err != nil {
+			log.Error().Err(err).Msg("Failed to release lock")
+		}
+	}()
 
 	common.UpdateRequestProgress(reqID, common.ProgressInfo{Title: "Setting vNet:" + resourceName, Time: time.Now()})
 
@@ -1297,6 +1303,7 @@ func CreateVmObject(wg *sync.WaitGroup, nsId string, mciId string, vmInfoData *m
 
 // CreateVm is func to create VM (option = "register" for register existing VM)
 func CreateVm(wg *sync.WaitGroup, nsId string, mciId string, vmInfoData *model.TbVmInfo, option string) error {
+	log.Info().Msgf("Start to create VM: %s", vmInfoData.Name)
 	//goroutin
 	defer wg.Done()
 

src/core/resource/common.go

@@ -989,18 +989,15 @@ func GetResource(nsId string, resourceType string, resourceId string) (interface
 				return nil, err
 			}
 
-			fmt.Printf("HTTP Status code: %d \n", resp.StatusCode())
 			switch {
 			case resp.StatusCode() >= 400 || resp.StatusCode() < 200:
 				err := fmt.Errorf(string(resp.Body()))
-				fmt.Println("body: ", string(resp.Body()))
 				log.Error().Err(err).Msg("")
 				return nil, err
 			}
 
 			updatedSpiderMyImage := resp.Result().(*model.SpiderMyImageInfo)
 			res.Status = updatedSpiderMyImage.Status
-			fmt.Printf("res.Status: %s \n", res.Status) // for debug
 			UpdateResourceObject(nsId, model.StrCustomImage, res)
 
 			return res, nil
@@ -1095,6 +1092,7 @@ func GenSpecMapKey(region, specName string) string {
 	return strings.ToLower(fmt.Sprintf("%s-%s", region, specName))
 }
 
+// GenResourceKey generates a Resource key for concatenating providerName, regionName, zoneName, resourceName
 func GetProviderRegionZoneResourceKey(providerName, regionName, zoneName, resourceName string) string {
 
 	div := "+"
@@ -1110,6 +1108,28 @@ func GetProviderRegionZoneResourceKey(providerName, regionName, zoneName, resour
 	return strings.ToLower(fmt.Sprintf("%s%s%s%s%s%s%s", providerName, div, regionName, div, zoneName, div, resourceName))
 }
 
+// ResolveProviderRegionZoneResourceKey resolves the Resource key into providerName, regionName, zoneName, resourceName
+func ResolveProviderRegionZoneResourceKey(key string) (providerName string, regionName string, zoneName string, resourceName string, err error) {
+
+	div := "+"
+
+	split := strings.Split(key, div)
+
+	if len(split) == 1 {
+		return "", "", "", "", fmt.Errorf("ResourceKey dose not contain div(%s)", div)
+	}
+
+	if len(split) == 2 {
+		return split[0], "", "", split[1], nil
+	}
+
+	if len(split) == 3 {
+		return split[0], split[1], "", split[2], nil
+	}
+
+	return split[0], split[1], split[2], split[3], nil
+}
+
 // CheckResource returns the existence of the TB Resource resource in bool form.
 func CheckResource(nsId string, resourceType string, resourceId string) (bool, error) {
 

src/core/resource/image.go

@@ -543,35 +543,83 @@ func GetImage(nsId string, imageKey string) (model.TbImageInfo, error) {
 	// make comparison case-insensitive
 	nsId = strings.ToLower(nsId)
 	imageKey = strings.ToLower(imageKey)
+	imageKey = strings.ReplaceAll(imageKey, " ", "")
 
-	// ex: tencent+ap-jakarta+ubuntu22.04
-	image := model.TbImageInfo{Namespace: nsId, Id: imageKey}
-	has, err := model.ORM.Where("LOWER(Namespace) = ? AND LOWER(Id) = ?", nsId, imageKey).Get(&image)
+	providerName, regionName, _, resourceName, err := ResolveProviderRegionZoneResourceKey(imageKey)
 	if err != nil {
-		log.Info().Err(err).Msgf("Failed to get image %s by ID", imageKey)
-	}
-	if has {
-		return image, nil
-	}
+		// imageKey does not include information for providerName, regionName
+		image := model.TbImageInfo{Namespace: nsId, Id: imageKey}
+
+		// 1) Check if the image is a custom image
+		// ex: custom-img-487zeit5
+		tempInterface, err := GetResource(nsId, model.StrCustomImage, imageKey)
+		customImage := model.TbCustomImageInfo{}
+		if err == nil {
+			err = common.CopySrcToDest(&tempInterface, &customImage)
+			if err != nil {
+				log.Error().Err(err).Msg("TbCustomImageInfo CopySrcToDest error")
+				return model.TbImageInfo{}, err
+			}
+			image.CspImageName = customImage.CspResourceName
+			image.SystemLabel = model.StrCustomImage
+			return image, nil
+		}
 
-	// ex: img-487zeit5
-	image = model.TbImageInfo{Namespace: nsId, CspImageName: imageKey}
-	has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(CspImageName) = ?", nsId, imageKey).Get(&image)
-	if err != nil {
-		log.Info().Err(err).Msgf("Failed to get image %s by CspImageName", imageKey)
-	}
-	if has {
-		return image, nil
-	}
+		// 2) Check if the image is a registered image in the given namespace
+		// ex: img-487zeit5
+		image = model.TbImageInfo{Namespace: nsId, Id: imageKey}
+		has, err := model.ORM.Where("LOWER(Namespace) = ? AND LOWER(Id) = ?", nsId, imageKey).Get(&image)
+		if err != nil {
+			log.Info().Err(err).Msgf("Cannot get image %s by ID from %s", imageKey, nsId)
+		}
+		if has {
+			return image, nil
+		}
+
+	} else {
+		// imageKey includes information for providerName, regionName
+
+		// 1) Check if the image is a registered image in the common namespace model.SystemCommonNs by ImageId
+		// ex: tencent+ap-jakarta+ubuntu22.04 or tencent+ap-jakarta+img-487zeit5
+		image := model.TbImageInfo{Namespace: model.SystemCommonNs, Id: imageKey}
+		has, err := model.ORM.Where("LOWER(Namespace) = ? AND LOWER(Id) = ?", model.SystemCommonNs, imageKey).Get(&image)
+		if err != nil {
+			log.Info().Err(err).Msgf("Cannot get image %s by ID from %s", imageKey, model.SystemCommonNs)
+		}
+		if has {
+			return image, nil
+		}
+
+		// 2) Check if the image is a registered image in the common namespace model.SystemCommonNs by CspImageName
+		// ex: tencent+ap-jakarta+img-487zeit5
+		image = model.TbImageInfo{Namespace: model.SystemCommonNs, CspImageName: resourceName}
+		has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(CspImageName) = ? AND LOWER(Id) LIKE ? AND LOWER(Id) LIKE ?",
+			model.SystemCommonNs,
+			resourceName,
+			"%"+strings.ToLower(providerName)+"%",
+			"%"+strings.ToLower(regionName)+"%").Get(&image)
+		if err != nil {
+			log.Info().Err(err).Msgf("Cannot get image %s by CspImageName", resourceName)
+		}
+		if has {
+			return image, nil
+		}
+
+		// 3) Check if the image is a registered image in the common namespace model.SystemCommonNs by GuestOS
+		// ex: tencent+ap-jakarta+Ubuntu22.04
+		image = model.TbImageInfo{Namespace: model.SystemCommonNs, GuestOS: resourceName}
+		has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(GuestOS) LIKE ? AND LOWER(Id) LIKE ? AND LOWER(Id) LIKE ?",
+			model.SystemCommonNs,
+			"%"+strings.ToLower(resourceName)+"%",
+			"%"+strings.ToLower(providerName)+"%",
+			"%"+strings.ToLower(regionName)+"%").Get(&image)
+		if err != nil {
+			log.Info().Err(err).Msgf("Failed to get image %s by GuestOS type", resourceName)
+		}
+		if has {
+			return image, nil
+		}
 
-	// ex: Ubuntu22.04
-	image = model.TbImageInfo{Namespace: nsId, GuestOS: imageKey}
-	has, err = model.ORM.Where("LOWER(Namespace) = ? AND LOWER(GuestOS) LIKE ?", nsId, imageKey).Get(&image)
-	if err != nil {
-		log.Info().Err(err).Msgf("Failed to get image %s by GuestOS type", imageKey)
-	}
-	if has {
-		return image, nil
 	}
 
 	return model.TbImageInfo{}, fmt.Errorf("The imageKey %s not found by any of ID, CspImageName, GuestOS", imageKey)