Add support for category-based manifest pre-processors

pkg/migration/configurationmetadata_steps.go

@@ -42,18 +42,18 @@ const (
 	errConfigurationMetadataOutput = "failed to output configuration JSON merge document"
 )
 
-func (pg *PlanGenerator) convertConfigurationMetadata(o UnstructuredWithMetadata) (*UnstructuredWithMetadata, bool, error) {
+func (pg *PlanGenerator) convertConfigurationMetadata(o UnstructuredWithMetadata) error {
 	isConverted := false
 	var conf metav1.Object
 	var err error
-	for _, confConv := range pg.registry.configurationConverters {
+	for _, confConv := range pg.registry.configurationMetaConverters {
 		if confConv.re == nil || confConv.converter == nil || !confConv.re.MatchString(o.Object.GetName()) {
 			continue
 		}
 
 		conf, err = toConfigurationMetadata(o.Object)
 		if err != nil {
-			return nil, false, err
+			return err
 		}
 		switch o.Object.GroupVersionKind().Version {
 		case "v1alpha1":
@@ -62,18 +62,21 @@ func (pg *PlanGenerator) convertConfigurationMetadata(o UnstructuredWithMetadata
 			err = confConv.converter.ConfigurationMetadataV1(conf.(*xpmetav1.Configuration))
 		}
 		if err != nil {
-			return nil, false, errors.Wrapf(err, "failed to call converter on Configuration: %s", conf.GetName())
+			return errors.Wrapf(err, "failed to call converter on Configuration: %s", conf.GetName())
 		}
 		// TODO: if a configuration converter only converts a specific version,
 		// (or does not convert the given configuration),
 		// we will have a false positive. Better to compute and check
 		// a diff here.
 		isConverted = true
 	}
-	return &UnstructuredWithMetadata{
+	if !isConverted {
+		return nil
+	}
+	return pg.stepEditConfigurationMetadata(o, &UnstructuredWithMetadata{
 		Object:   ToSanitizedUnstructured(conf),
 		Metadata: o.Metadata,
-	}, isConverted, nil
+	})
 }
 
 func (pg *PlanGenerator) stepConfiguration(s step) *Step {

pkg/migration/converter.go

@@ -21,6 +21,7 @@ import (
 	xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1"
 	xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1"
 	xppkgv1 "github.com/crossplane/crossplane/apis/pkg/v1"
+	xppkgv1beta1 "github.com/crossplane/crossplane/apis/pkg/v1beta1"
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -35,9 +36,9 @@ const (
 	errFromUnstructuredConfMeta    = "failed to convert from unstructured.Unstructured to Crossplane Configuration metadata"
 	errFromUnstructuredConfPackage = "failed to convert from unstructured.Unstructured to Crossplane Configuration package"
 	errFromUnstructuredProvider    = "failed to convert from unstructured.Unstructured to Crossplane Provider package"
-	// errFromUnstructuredLock     = "failed to convert from unstructured.Unstructured to Crossplane package lock"
-	errToUnstructured        = "failed to convert from the managed resource type to unstructured.Unstructured"
-	errRawExtensionUnmarshal = "failed to unmarshal runtime.RawExtension"
+	errFromUnstructuredLock        = "failed to convert from unstructured.Unstructured to Crossplane package lock"
+	errToUnstructured              = "failed to convert from the managed resource type to unstructured.Unstructured"
+	errRawExtensionUnmarshal       = "failed to unmarshal runtime.RawExtension"
 
 	errFmtPavedDelete = "failed to delete fieldpath %q from paved"
 )
@@ -145,10 +146,12 @@ func FromGroupVersionKind(gvk schema.GroupVersionKind) GroupVersionKind {
 	}
 }
 
-// workaround for:
+// ToComposition converts the specified unstructured.Unstructured to
+// a Crossplane Composition.
+// Workaround for:
 // https://github.com/kubernetes-sigs/structured-merge-diff/issues/230
-func convertToComposition(u map[string]interface{}) (*xpv1.Composition, error) {
-	buff, err := json.Marshal(u)
+func ToComposition(u unstructured.Unstructured) (*xpv1.Composition, error) {
+	buff, err := json.Marshal(u.Object)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to marshal map to JSON")
 	}
@@ -240,10 +243,19 @@ func toProviderPackage(u unstructured.Unstructured) (*xppkgv1.Provider, error) {
 	return pkg, nil
 }
 
-/*func toPackageLock(u unstructured.Unstructured) (*xppkgv1beta1.Lock, error) {
+func getCategory(u unstructured.Unstructured) Category {
+	switch u.GroupVersionKind() {
+	case xpv1.CompositionGroupVersionKind:
+		return CategoryComposition
+	default:
+		return categoryUnknown
+	}
+}
+
+func toPackageLock(u unstructured.Unstructured) (*xppkgv1beta1.Lock, error) {
 	lock := &xppkgv1beta1.Lock{}
 	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, lock); err != nil {
 		return nil, errors.Wrap(err, errFromUnstructuredLock)
 	}
 	return lock, nil
-}*/
+}

pkg/migration/filesystem.go

@@ -13,6 +13,11 @@ import (
 	sigsyaml "sigs.k8s.io/yaml"
 )
 
+var (
+	_ Source = &FileSystemSource{}
+	_ Target = &FileSystemTarget{}
+)
+
 // FileSystemSource is a source implementation to read resources from filesystem
 type FileSystemSource struct {
 	index int
@@ -62,7 +67,8 @@ func NewFileSystemSource(dir string, opts ...FileSystemSourceOption) (*FileSyste
 		fs.items = append(fs.items, UnstructuredWithMetadata{
 			Object: *u,
 			Metadata: Metadata{
-				Path: path,
+				Path:     path,
+				Category: getCategory(*u),
 			},
 		})
 
@@ -89,6 +95,12 @@ func (fs *FileSystemSource) Next() (UnstructuredWithMetadata, error) {
 	return UnstructuredWithMetadata{}, errors.New("no more elements")
 }
 
+// Reset resets the source so that resources can be reread from the beginning.
+func (fs *FileSystemSource) Reset() error {
+	fs.index = 0
+	return nil
+}
+
 // FileSystemTarget is a target implementation to write/patch/delete resources to file system
 type FileSystemTarget struct {
 	afero afero.Afero

pkg/migration/interfaces.go

@@ -120,6 +120,11 @@ type Source interface {
 	// Next returns the next resource manifest available or
 	// any errors encountered while reading the next resource manifest.
 	Next() (UnstructuredWithMetadata, error)
+	// Reset resets the Source so that it can read the manifests
+	// from the beginning. There is no guarantee that the Source
+	// will return the same set of manifests or it will return
+	// them in the same order after a reset.
+	Reset() error
 }
 
 // Target is a target where resource manifests can be manipulated
@@ -145,3 +150,14 @@ type Executor interface {
 	// executing the plan.
 	Destroy() error
 }
+
+// UnstructuredPreProcessor allows manifests read by the Source
+// to be pre-processed before the converters are run
+// It's not possible to do any conversions via the pre-processors,
+// and they only allow migrators to extract information from
+// the manifests read by the Source before any converters are run.
+type UnstructuredPreProcessor interface {
+	// PreProcess is called for a manifest read by the Source
+	// before any converters are run.
+	PreProcess(u UnstructuredWithMetadata) error
+}