feat: cloud-hypervisor support

Flintlock will default to Firecracker for creating microvms but you can:
- change the default to Cloud Hypervisor when starting Flintlock
- specify on a per VM basis whether to use Firecracker or Cloud Hypervisor

A couple of things to note:
- Cloud Hypervisor supports macvtap so you don't have to use the
  Weaveworks fork
- Cloud Hypervisor doesn't have a metadata service so you need to find another solution if you rely on this. For cloud-init we attach an additional volume.
- Cloud Hypervisor supports pci-passtrhough, vDPA etc. These features aren't currently exposed via the API but they will be in the future.

Signed-off-by: Richard Case <richard.case@outlook.com>

README.md

@@ -7,18 +7,21 @@
 
 ## What is flintlock?
 
-Flintlock is a service for creating and managing the lifecycle of microVMs on a host machine. Initially we will be supporting [Firecracker](https://firecracker-microvm.github.io/).
+Flintlock is a service for creating and managing the lifecycle of microVMs on a host machine. We support [Firecracker](https://firecracker-microvm.github.io/) and [Cloud Hypervisor](https://www.cloudhypervisor.org/) (experimental).
 
-The primary use case for flintlock is to create microVMs on a bare-metal host where the microVMs will be used as nodes in a virtualized Kubernetes cluster. It is an essential part of [Liquid Metal](https://www.weave.works/blog/multi-cluster-kubernetes-on-microvms-for-bare-metal) and will ultimately be driven by Cluster API Provider Microvm (coming soon).
+The original use case for flintlock was to create microVMs on a bare-metal host where the microVMs will be used as nodes in a virtualized Kubernetes cluster. It is an essential part of [Liquid Metal](https://www.weave.works/blog/multi-cluster-kubernetes-on-microvms-for-bare-metal) and can be orchestrated by [Cluster API Provider Microvm](https://github.com/weaveworks-liquidmetal/cluster-api-provider-microvm).
+
+However, its useful for many other use cases where lightweight virtualization is required (e.g. isolated workloads, pipelines).
 
 ## Features
 
 Using API requests (via gRPC or HTTP):
 
-- Create and delete microVMs using Firecracker
+- Create and delete microVMs
 - Manage the lifecycle of microVMs (i.e. start, stop, pause)
 - Configure microVM metadata via cloud-init, ignition etc
 - Use OCI images for microVM volumes, kernel and initrd
+- Expose microVM metrics for collection by Prometheus
 - (coming soon) Use CNI to configure the network for the microVMs
 
 ## Documentation
@@ -50,13 +53,14 @@ Your feedback is always welcome!
 
 The table below shows you which versions of Firecracker are compatible with Flintlock:
 
-| Flintlock         | Firecracker                        |
-| ----------------- | ---------------------------------- |
-| >= v0.3.0         | Official v1.0+ or >=v1.0.0-macvtap |
-| <= v0.2.0         | <= v0.25.2-macvtap                 |
-| <= v0.1.0-alpha.6 | <= v0.25.2-macvtap                 |
-|    v0.1.0-alpha.7 | **Do not use**                     |
-|    v0.1.0-alpha.8 | <= v0.25.2-macvtap                 |
+| Flintlock         | Firecracker                      | Cloud Hypervisor  |
+| ----------------- | -------------------------------- | ----------------- |
+|    v0.4.0         | Official v1.0+ or v1.0.0-macvtap | v26.0             |
+|    v0.3.0         | Official v1.0+ or v1.0.0-macvtap | **Not Supported** |
+| <= v0.2.0         | <= v0.25.2-macvtap               | **Not Supported** |
+| <= v0.1.0-alpha.6 | <= v0.25.2-macvtap               | **Not Supported** |
+|    v0.1.0-alpha.7 | **Do not use**                   | **Not Supported** |
+|    v0.1.0-alpha.8 | <= v0.25.2-macvtap               | **Not Supported** |
 
 > Note: Flintlock currently requires a custom build of Firecracker if you plan to use `macvtap` available [here][fc-fork].
 

api/services/microvm/v1alpha1/microvms.swagger.json

@@ -341,6 +341,10 @@
         "uid": {
           "type": "string",
           "description": "UID is a globally unique identifier of the microvm."
+        },
+        "provider": {
+          "type": "string",
+          "description": "Provider allows you to specify the name of the microvm provider to use. If this isn't supplied\nthen the default provider will be used."
         }
       },
       "description": "MicroVMSpec represents the specification for a microvm."

api/types/microvm.proto

@@ -66,6 +66,9 @@ message MicroVMSpec {
   // UID is a globally unique identifier of the microvm.
   optional string uid = 15;
 
+  // Provider allows you to specify the name of the microvm provider to use. If this isn't supplied
+  // then the default provider will be used.
+  optional string provider = 16;
 }
 
 // Kernel represents the configuration for a kernel.

buf.lock

@@ -4,8 +4,8 @@ deps:
   - remote: buf.build
     owner: googleapis
     repository: googleapis
-    commit: 783e4b5374fa488ab068d08af9658438
+    commit: 75b4300737fb4efca0831636be94e517
   - remote: buf.build
     owner: grpc-ecosystem
     repository: grpc-gateway
-    commit: b96615cde70c403f8075c48e56178f88
+    commit: a1ecdc58eccd49aa8bea2a7a9022dc27

client/cloudinit/cloudinit.go

@@ -7,4 +7,8 @@ const (
 	UserdataKey = "user-data"
 	// VendorDataKey is the metadata key name to use for vendor data.
 	VendorDataKey = "vendor-data"
+	// NetworkConfigDataKey is the metadata key name for the network config.
+	NetworkConfigDataKey = "network-config"
+	// VolumeName is the name of a volume that contains cloud-init data.
+	VolumeName = "cidata"
 )

core/application/app.go

@@ -25,6 +25,7 @@ type app struct {
 }
 
 type Config struct {
-	RootStateDir string
-	MaximumRetry int
+	RootStateDir    string
+	MaximumRetry    int
+	DefaultProvider string
 }

core/application/app_test.go

@@ -47,6 +47,9 @@ func TestApp_CreateMicroVM(t *testing.T) {
 			expectError:  false,
 			expect: func(rm *mock.MockMicroVMRepositoryMockRecorder, em *mock.MockEventServiceMockRecorder, im *mock.MockIDServiceMockRecorder, pm *mock.MockMicroVMServiceMockRecorder) {
 				im.GenerateRandom().Return("id1234", nil).Times(1)
+
+				pm.Capabilities().Return(models.Capabilities{models.MetadataServiceCapability})
+
 				im.GenerateRandom().Return(testUID, nil).Times(1)
 
 				rm.Get(
@@ -59,6 +62,7 @@ func TestApp_CreateMicroVM(t *testing.T) {
 				).Return(nil, nil)
 
 				expectedCreatedSpec := createTestSpecWithMetadata("id1234", defaults.MicroVMNamespace, testUID, createInstanceMetadatadata(t, testUID))
+				expectedCreatedSpec.Spec.Provider = "mock"
 				expectedCreatedSpec.Spec.CreatedAt = frozenTime().Unix()
 				expectedCreatedSpec.Status.State = models.PendingState
 
@@ -86,6 +90,7 @@ func TestApp_CreateMicroVM(t *testing.T) {
 			specToCreate: createTestSpec("id1234", "default", testUID),
 			expectError:  false,
 			expect: func(rm *mock.MockMicroVMRepositoryMockRecorder, em *mock.MockEventServiceMockRecorder, im *mock.MockIDServiceMockRecorder, pm *mock.MockMicroVMServiceMockRecorder) {
+				pm.Capabilities().Return(models.Capabilities{models.MetadataServiceCapability})
 				im.GenerateRandom().Return(testUID, nil).Times(1)
 				rm.Get(
 					gomock.AssignableToTypeOf(context.Background()),
@@ -100,6 +105,7 @@ func TestApp_CreateMicroVM(t *testing.T) {
 				)
 
 				expectedCreatedSpec := createTestSpecWithMetadata("id1234", "default", testUID, createInstanceMetadatadata(t, testUID))
+				expectedCreatedSpec.Spec.Provider = "mock"
 				expectedCreatedSpec.Spec.CreatedAt = frozenTime().Unix()
 				expectedCreatedSpec.Status.State = models.PendingState
 
@@ -146,6 +152,7 @@ func TestApp_CreateMicroVM(t *testing.T) {
 			specToCreate: createTestSpecWithMetadata("id1234", "default", testUID, createInstanceMetadatadata(t, "abcdef")),
 			expectError:  false,
 			expect: func(rm *mock.MockMicroVMRepositoryMockRecorder, em *mock.MockEventServiceMockRecorder, im *mock.MockIDServiceMockRecorder, pm *mock.MockMicroVMServiceMockRecorder) {
+				pm.Capabilities().Return(models.Capabilities{models.MetadataServiceCapability})
 				im.GenerateRandom().Return(testUID, nil).Times(1)
 				rm.Get(
 					gomock.AssignableToTypeOf(context.Background()),
@@ -160,6 +167,7 @@ func TestApp_CreateMicroVM(t *testing.T) {
 				)
 
 				expectedCreatedSpec := createTestSpecWithMetadata("id1234", "default", testUID, createInstanceMetadatadata(t, "abcdef"))
+				expectedCreatedSpec.Spec.Provider = "mock"
 				expectedCreatedSpec.Spec.CreatedAt = frozenTime().Unix()
 				expectedCreatedSpec.Status.State = models.PendingState
 
@@ -199,8 +207,10 @@ func TestApp_CreateMicroVM(t *testing.T) {
 			is := mock.NewMockImageService(mockCtrl)
 			fs := afero.NewMemMapFs()
 			ports := &ports.Collection{
-				Repo:              rm,
-				Provider:          pm,
+				Repo: rm,
+				MicrovmProviders: map[string]ports.MicroVMService{
+					"mock": pm,
+				},
 				EventService:      em,
 				IdentifierService: im,
 				NetworkService:    ns,
@@ -212,7 +222,7 @@ func TestApp_CreateMicroVM(t *testing.T) {
 			tc.expect(rm.EXPECT(), em.EXPECT(), im.EXPECT(), pm.EXPECT())
 
 			ctx := context.Background()
-			app := application.New(&application.Config{}, ports)
+			app := application.New(&application.Config{DefaultProvider: "mock"}, ports)
 			_, err := app.CreateMicroVM(ctx, tc.specToCreate)
 
 			if tc.expectError {
@@ -311,8 +321,10 @@ func TestApp_DeleteMicroVM(t *testing.T) {
 			is := mock.NewMockImageService(mockCtrl)
 			fs := afero.NewMemMapFs()
 			ports := &ports.Collection{
-				Repo:              rm,
-				Provider:          pm,
+				Repo: rm,
+				MicrovmProviders: map[string]ports.MicroVMService{
+					"mock": pm,
+				},
 				EventService:      em,
 				IdentifierService: im,
 				NetworkService:    ns,
@@ -324,7 +336,7 @@ func TestApp_DeleteMicroVM(t *testing.T) {
 			tc.expect(rm.EXPECT(), em.EXPECT(), im.EXPECT(), pm.EXPECT())
 
 			ctx := context.Background()
-			app := application.New(&application.Config{}, ports)
+			app := application.New(&application.Config{DefaultProvider: "mock"}, ports)
 			err := app.DeleteMicroVM(ctx, tc.toDeleteUID)
 
 			if tc.expectError {
@@ -417,8 +429,10 @@ func TestApp_GetMicroVM(t *testing.T) {
 			is := mock.NewMockImageService(mockCtrl)
 			fs := afero.NewMemMapFs()
 			ports := &ports.Collection{
-				Repo:              rm,
-				Provider:          pm,
+				Repo: rm,
+				MicrovmProviders: map[string]ports.MicroVMService{
+					"mock": pm,
+				},
 				EventService:      em,
 				IdentifierService: im,
 				NetworkService:    ns,
@@ -430,7 +444,7 @@ func TestApp_GetMicroVM(t *testing.T) {
 			tc.expect(rm.EXPECT(), em.EXPECT(), im.EXPECT(), pm.EXPECT())
 
 			ctx := context.Background()
-			app := application.New(&application.Config{}, ports)
+			app := application.New(&application.Config{DefaultProvider: "mock"}, ports)
 			mvm, err := app.GetMicroVM(ctx, tc.toGetUID)
 
 			if tc.expectError {
@@ -560,8 +574,10 @@ func TestApp_GetAllMicroVM(t *testing.T) {
 			is := mock.NewMockImageService(mockCtrl)
 			fs := afero.NewMemMapFs()
 			ports := &ports.Collection{
-				Repo:              rm,
-				Provider:          pm,
+				Repo: rm,
+				MicrovmProviders: map[string]ports.MicroVMService{
+					"mock": pm,
+				},
 				EventService:      em,
 				IdentifierService: im,
 				NetworkService:    ns,
@@ -573,7 +589,7 @@ func TestApp_GetAllMicroVM(t *testing.T) {
 			tc.expect(rm.EXPECT(), em.EXPECT(), im.EXPECT(), pm.EXPECT())
 
 			ctx := context.Background()
-			app := application.New(&application.Config{}, ports)
+			app := application.New(&application.Config{DefaultProvider: "mock"}, ports)
 			query := models.ListMicroVMQuery{"namespace": tc.toGetNS}
 
 			if tc.toGetName != nil {

core/application/commands.go

@@ -50,6 +50,14 @@ func (a *app) CreateMicroVM(ctx context.Context, mvm *models.MicroVM) (*models.M
 		mvm.ID = *vmid
 	}
 
+	if mvm.Spec.Provider == "" {
+		mvm.Spec.Provider = a.cfg.DefaultProvider
+	}
+	provider, ok := a.ports.MicrovmProviders[mvm.Spec.Provider]
+	if !ok {
+		return nil, fmt.Errorf("microvm provider %s isn't available", mvm.Spec.Provider)
+	}
+
 	uid, err := a.ports.IdentifierService.GenerateRandom()
 	if err != nil {
 		return nil, fmt.Errorf("generating random ID for microvm: %w", err)
@@ -81,7 +89,9 @@ func (a *app) CreateMicroVM(ctx context.Context, mvm *models.MicroVM) (*models.M
 	if err != nil {
 		return nil, fmt.Errorf("adding instance data: %w", err)
 	}
-	a.addMetadataInterface(mvm)
+	if provider.Capabilities().Has(models.MetadataServiceCapability) {
+		a.addMetadataInterface(mvm)
+	}
 
 	// Set the timestamp when the VMspec was created.
 	mvm.Spec.CreatedAt = a.ports.Clock().Unix()

core/errors/errors.go

@@ -123,3 +123,25 @@ func IsSpecNotFound(err error) bool {
 
 	return errors.As(err, e)
 }
+
+func NewNotSupported(featureName string) error {
+	return notSupportedError{
+		unsupported: featureName,
+	}
+}
+
+type notSupportedError struct {
+	unsupported string
+}
+
+// Error returns the error message.
+func (e notSupportedError) Error() string {
+	return fmt.Sprintf("%s is not supported", e.unsupported)
+}
+
+// IsNotSupported tests an error to see if its a not supported error.
+func IsNotSupported(err error) bool {
+	e := &notSupportedError{}
+
+	return errors.As(err, e)
+}

core/models/capability.go

@@ -6,7 +6,11 @@ type Capability string
 const (
 	// MetadataServiceCapability is a capability that indicates the microvm provider
 	// has a metadata service.
-	MetadataServiceCapability = "metadata-service"
+	MetadataServiceCapability Capability = "metadata-service"
+
+	// StartCapability is a capability that the microvm provider must be started separately from creation.
+	// If a provider doesn't have this capability then its assumed the microvm will be started at creation.
+	StartCapability Capability = "start"
 )
 
 // Capabilities represents a list of capabilities.

core/models/microvm.go

@@ -25,6 +25,8 @@ type MicroVM struct {
 
 // MicroVMSpec represents the specification of a microvm machine.
 type MicroVMSpec struct {
+	// Provider specifies the name of the microvm provider to use.
+	Provider string `json:"provider"`
 	// Kernel specifies the kernel and its argments to use.
 	Kernel Kernel `json:"kernel" validate:"omitempty"`
 	// Initrd is an optional initial ramdisk to use.

core/plans/helper_test.go

@@ -37,11 +37,13 @@ func fakePorts(mockCtrl *gomock.Controller) (*mockList, *ports.Collection) {
 		Repo:              mList.MicroVMRepository,
 		EventService:      mList.EventService,
 		IdentifierService: mList.IDService,
-		Provider:          mList.MicroVMService,
-		NetworkService:    mList.NetworkService,
-		ImageService:      mList.ImageService,
-		FileSystem:        afero.NewMemMapFs(),
-		Clock:             time.Now,
+		MicrovmProviders: map[string]ports.MicroVMService{
+			"mock": mList.MicroVMService,
+		},
+		NetworkService: mList.NetworkService,
+		ImageService:   mList.ImageService,
+		FileSystem:     afero.NewMemMapFs(),
+		Clock:          time.Now,
 	}
 }
 
@@ -66,6 +68,7 @@ func createTestSpec(name, ns string) *models.MicroVM {
 			},
 		},
 		Spec: models.MicroVMSpec{
+			Provider:   "mock",
 			VCPU:       2,
 			MemoryInMb: 2048,
 			Kernel: models.Kernel{

core/plans/microvm_create_update.go

@@ -48,6 +48,11 @@ func (p *microvmCreateOrUpdatePlan) Create(ctx context.Context) ([]planner.Proce
 		return nil, portsctx.ErrPortsMissing
 	}
 
+	provider, ok := ports.MicrovmProviders[p.vm.Spec.Provider]
+	if !ok {
+		return nil, fmt.Errorf("microvm provider %s isn't available", p.vm.Spec.Provider)
+	}
+
 	if p.vm.Spec.DeletedAt != 0 {
 		return []planner.Procedure{}, nil
 	}
@@ -70,13 +75,15 @@ func (p *microvmCreateOrUpdatePlan) Create(ctx context.Context) ([]planner.Proce
 	}
 
 	// MicroVM provider create
-	if err := p.addStep(ctx, microvm.NewCreateStep(p.vm, ports.Provider)); err != nil {
+	if err := p.addStep(ctx, microvm.NewCreateStep(p.vm, provider)); err != nil {
 		return nil, fmt.Errorf("adding microvm create step: %w", err)
 	}
 
 	// MicroVM provider start
-	if err := p.addStep(ctx, microvm.NewStartStep(p.vm, ports.Provider, microVMBootTime)); err != nil {
-		return nil, fmt.Errorf("adding microvm start step: %w", err)
+	if provider.Capabilities().Has(models.StartCapability) {
+		if err := p.addStep(ctx, microvm.NewStartStep(p.vm, provider, microVMBootTime)); err != nil {
+			return nil, fmt.Errorf("adding microvm start step: %w", err)
+		}
 	}
 
 	return p.steps, nil

core/plans/microvm_create_update_test.go

@@ -41,6 +41,8 @@ func TestMicroVMCreateOrUpdatePlan(t *testing.T) {
 		EXPECT().
 		Create(gomock.Any(), gomock.Any())
 
+	mList.MicroVMService.EXPECT().Capabilities().Return(models.Capabilities{models.StartCapability})
+
 	mList.MicroVMService.
 		EXPECT().
 		Start(gomock.Any(), gomock.Any()).

core/plans/microvm_delete.go

@@ -52,14 +52,19 @@ func (p *microvmDeletePlan) Create(ctx context.Context) ([]planner.Procedure, er
 		return nil, portsctx.ErrPortsMissing
 	}
 
+	provider, ok := ports.MicrovmProviders[p.vm.Spec.Provider]
+	if !ok {
+		return nil, fmt.Errorf("microvm provider %s isn't available", p.vm.Spec.Provider)
+	}
+
 	if p.vm.Spec.DeletedAt == 0 {
 		return []planner.Procedure{}, nil
 	}
 
 	p.clearPlanList()
 
 	// MicroVM provider delete
-	if err := p.addStep(ctx, microvm.NewDeleteStep(p.vm, ports.Provider)); err != nil {
+	if err := p.addStep(ctx, microvm.NewDeleteStep(p.vm, provider)); err != nil {
 		return nil, fmt.Errorf("adding microvm delete step: %w", err)
 	}