From 111782119bf9d20a0ab745e424a354b134312e31 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Fri, 26 Oct 2018 16:36:37 -0700 Subject: [PATCH] rhcos: implement image discovery for new pipeline RHCOS has moved over to the 2.0 pipeline which has a nicer discovery mechanism for images. This uses that new pipeline to fetch the latest QCOW images for the latest build. Unfortunately, there is no "latest" alias anymore, so the installer has to fetch resources from the Internet before it can prompt the user. This will go away once the installer starts pinning to a specific RHCOS release. --- pkg/asset/installconfig/platform.go | 16 ++++- pkg/rhcos/ami.go | 80 +++------------------ pkg/rhcos/builds.go | 108 ++++++++++++++++++++++++++++ pkg/rhcos/qemu.go | 18 +++++ 4 files changed, 148 insertions(+), 74 deletions(-) create mode 100644 pkg/rhcos/builds.go create mode 100644 pkg/rhcos/qemu.go diff --git a/pkg/asset/installconfig/platform.go b/pkg/asset/installconfig/platform.go index cac8284b6d9..00f3daec937 100644 --- a/pkg/asset/installconfig/platform.go +++ b/pkg/asset/installconfig/platform.go @@ -1,6 +1,7 @@ package installconfig import ( + "context" "encoding/json" "fmt" "net/url" @@ -12,6 +13,7 @@ import ( survey "gopkg.in/AlecAivazis/survey.v1" "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types" ) @@ -52,7 +54,6 @@ var ( defaultLibvirtNetworkIfName = "tt0" defaultLibvirtNetworkIPRange = "192.168.126.0/24" - defaultLibvirtImageURL = "http://aos-ostree.rhev-ci-vms.eng.rdu2.redhat.com/rhcos/images/cloud/latest/rhcos-qemu.qcow2.gz" ) // Platform is an asset that queries the user for the platform on which to install @@ -288,13 +289,24 @@ func (a *platform) libvirtPlatform() (*types.LibvirtPlatform, error) { return nil, err } + // TODO: Ideally, this would live inside of a closure which is passed to + // asset.GenerateUserProvidedAsset and only called if the environment + // variable isn't present. As this exists, it ruins the abstraction. + var qcowImage string + if _, ok := os.LookupEnv("OPENSHIFT_INSTALL_LIBVIRT_IMAGE"); !ok { + qcowImage, err = rhcos.QEMU(context.TODO(), rhcos.DefaultChannel) + if err != nil { + return nil, errors.Wrap(err, "failed to fetch QEMU image URL") + } + } + image, err := asset.GenerateUserProvidedAsset( "Libvirt Image", &survey.Question{ Prompt: &survey.Input{ Message: "Image", Help: "URI of the OS image.", - Default: defaultLibvirtImageURL, + Default: qcowImage, }, Validate: survey.ComposeValidators(survey.Required, uriValidator), }, diff --git a/pkg/rhcos/ami.go b/pkg/rhcos/ami.go index 3f653ef8d15..2726e4c6fcd 100644 --- a/pkg/rhcos/ami.go +++ b/pkg/rhcos/ami.go @@ -2,86 +2,22 @@ package rhcos import ( "context" - "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" "github.com/pkg/errors" ) -const ( - // DefaultChannel is the default RHCOS channel for the cluster. - DefaultChannel = "tested" -) - -// AMI calculates a Red Hat CoreOS AMI. -func AMI(ctx context.Context, channel, region string) (ami string, err error) { - if channel != DefaultChannel { - return "", errors.Errorf("channel %q is not yet supported", channel) - } - - ssn := session.Must(session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, - Config: aws.Config{ - Region: aws.String(region), - }, - })) - - svc := ec2.New(ssn) - - result, err := svc.DescribeImagesWithContext(ctx, &ec2.DescribeImagesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("name"), - Values: aws.StringSlice([]string{"redhat-coreos-*"}), - }, - { - Name: aws.String("architecture"), - Values: aws.StringSlice([]string{"x86_64"}), - }, - { - Name: aws.String("virtualization-type"), - Values: aws.StringSlice([]string{"hvm"}), - }, - { - Name: aws.String("image-type"), - Values: aws.StringSlice([]string{"machine"}), - }, - { - Name: aws.String("owner-id"), - Values: aws.StringSlice([]string{"531415883065"}), - }, - { - Name: aws.String("state"), - Values: aws.StringSlice([]string{"available"}), - }, - }, - }) +// AMI fetches the HVM AMI ID of the latest Red Hat CoreOS release. +func AMI(ctx context.Context, channel, region string) (string, error) { + meta, err := fetchLatestMetadata(ctx, channel) if err != nil { - return "", errors.Wrap(err, "failed to describe AMIs") + return "", errors.Wrap(err, "failed to fetch RHCOS metadata") } - var image *ec2.Image - var created time.Time - for _, nextImage := range result.Images { - if nextImage.ImageId == nil || nextImage.CreationDate == nil { - continue + for _, ami := range meta.AMIs { + if ami.Name == region { + return ami.HVM, nil } - nextCreated, err := time.Parse(time.RFC3339, *nextImage.CreationDate) - if err != nil { - return "", errors.Wrap(err, "failed to parse AMIs CreationDate to time.RFC3339") - } - - if image == nil || nextCreated.After(created) { - image = nextImage - created = nextCreated - } - } - - if image == nil { - return "", errors.Errorf("no RHCOS AMIs found in %s", region) } - return *image.ImageId, nil + return "", errors.Errorf("no RHCOS AMIs found in %s", region) } diff --git a/pkg/rhcos/builds.go b/pkg/rhcos/builds.go new file mode 100644 index 00000000000..a1a76158d9c --- /dev/null +++ b/pkg/rhcos/builds.go @@ -0,0 +1,108 @@ +package rhcos + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + // DefaultChannel is the default RHCOS channel for the cluster. + DefaultChannel = "maipo" + + baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases" +) + +type metadata struct { + AMIs []struct { + HVM string `json:"hvm"` + Name string `json:"name"` + } `json:"amis"` + Images struct { + QEMU struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + } `json:"qemu"` + } `json:"images"` + OSTreeVersion string `json:"ostree-version"` +} + +func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) { + build, err := fetchLatestBuild(ctx, channel) + if err != nil { + return metadata{}, errors.Wrap(err, "failed to fetch latest build") + } + + url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build) + logrus.Debugf("Fetching RHCOS metadata from %q", url) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return metadata{}, errors.Wrap(err, "failed to build request") + } + + client := &http.Client{} + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return metadata{}, errors.Wrap(err, "failed to fetch metadata") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return metadata{}, errors.Wrap(err, "failed to read HTTP response") + } + + var meta metadata + if err := json.Unmarshal(body, &meta); err != nil { + return meta, errors.Wrap(err, "failed to parse HTTP response") + } + + return meta, nil +} + +func fetchLatestBuild(ctx context.Context, channel string) (string, error) { + url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel) + logrus.Debugf("Fetching RHCOS builds from %q", url) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", errors.Wrap(err, "failed to build request") + } + + client := &http.Client{} + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return "", errors.Wrap(err, "failed to fetch builds") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.Errorf("incorrect HTTP response (%s)", resp.Status) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "failed to read HTTP response") + } + + var builds struct { + Builds []string `json:"builds"` + } + if err := json.Unmarshal(body, &builds); err != nil { + return "", errors.Wrap(err, "failed to parse HTTP response") + } + + if len(builds.Builds) == 0 { + return "", errors.Errorf("no builds found") + } + + return builds.Builds[0], nil +} diff --git a/pkg/rhcos/qemu.go b/pkg/rhcos/qemu.go new file mode 100644 index 00000000000..8fd30de8c91 --- /dev/null +++ b/pkg/rhcos/qemu.go @@ -0,0 +1,18 @@ +package rhcos + +import ( + "context" + "fmt" + + "github.com/pkg/errors" +) + +// QEMU fetches the URL of the latest Red Hat CoreOS release. +func QEMU(ctx context.Context, channel string) (string, error) { + meta, err := fetchLatestMetadata(ctx, channel) + if err != nil { + return "", errors.Wrap(err, "failed to fetch RHCOS metadata") + } + + return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil +}