Skip to content

Commit

Permalink
ROX-11407: Support for Configurable Resource Limits for Central & Sca…
Browse files Browse the repository at this point in the history
…nner (#243)

Allow resource settings (requests and limits) to be configured during central creation through the Admin API.
  • Loading branch information
mtesseract committed Aug 9, 2022
1 parent 2632f90 commit 4f29bb2
Show file tree
Hide file tree
Showing 34 changed files with 1,415 additions and 239 deletions.
1 change: 1 addition & 0 deletions .openshift-ci/tests/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then
export "${secret_name}"="${secret_value}"
done
export STATIC_TOKEN="${FLEET_STATIC_TOKEN:-}"
export STATIC_TOKEN_ADMIN="${FLEET_STATIC_TOKEN_ADMIN:-}"
export QUAY_USER="${IMAGE_PUSH_USERNAME:-}"
export QUAY_TOKEN="${IMAGE_PUSH_PASSWORD:-}"
export CLUSTER_TYPE="openshift-ci"
Expand Down
6 changes: 3 additions & 3 deletions .secrets.baseline
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,7 @@
"filename": "openapi/fleet-manager-private.yaml",
"hashed_secret": "2774b5ad0fae1c8b8dc897b89db85529d45cb5cf",
"is_verified": false,
"line_number": 472,
"line_number": 474,
"is_secret": false
}
],
Expand All @@ -503,7 +503,7 @@
"filename": "openapi/fleet-manager.yaml",
"hashed_secret": "a5f0056c7d0ca4ed78e2f3b551554daaf4721934",
"is_verified": false,
"line_number": 1037,
"line_number": 1094,
"is_secret": false
}
],
Expand Down Expand Up @@ -834,5 +834,5 @@
}
]
},
"generated_at": "2022-08-05T13:45:51Z"
"generated_at": "2022-08-09T08:36:37Z"
}
14 changes: 14 additions & 0 deletions dev/env/manifests/shared/03-configmap-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,11 @@ data:
any_user: true
max_allowed_instances: 3
registered_users: []
# Static token's org_id, see config/static-token-payload.json
- id: 12345678
any_user: true
max_allowed_instances: 100
registered_users: []
read-only-user-list.yaml: |-
---
# A list of users with read-only permissions for data plane clusters
Expand All @@ -227,6 +232,11 @@ data:
roles:
- "acs-general-engineering" # Will include all of ACS engineering. Available also within staging environment.
- "acs-fleet-manager-admin-full" # Prod rover group, will only include selected members + SREs.
- method: POST
roles:
- "acs-general-engineering" # Will include all of ACS engineering. Available also within staging environment.
- "acs-fleet-manager-admin-full" # Prod rover group, will only include selected members + SREs.
- "acs-fleet-manager-admin-write" # Prod rover group, will only include selected members + SREs.
admin-authz-roles-prod.yaml: |-
---
- method: GET
Expand All @@ -241,6 +251,10 @@ data:
- method: DELETE
roles:
- "acs-fleet-manager-admin-full" # Prod rover group, will only include selected members + SREs.
- method: POST
roles:
- "acs-fleet-manager-admin-full" # Prod rover group, will only include selected members + SREs.
- "acs-fleet-manager-admin-write" # Prod rover group, will only include selected members + SREs.
kind: ConfigMap
metadata:
name: config
Expand Down
111 changes: 111 additions & 0 deletions e2e/admin_client_api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package e2e

import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"

"github.com/golang/glog"
"github.com/pkg/errors"
"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/fleetmanager"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/compat"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/public"
)

// Client represents the REST client for connecting to fleet-manager
type Client struct {
client http.Client
auth fleetmanager.Auth
adminAPIEndpoint string
}

func (c *Client) newRequest(method string, url string, body io.Reader) (*http.Response, error) {
glog.Infof("Send request to %s", url)
r, err := http.NewRequest(method, url, body)
if err != nil {
return nil, fmt.Errorf("building HTTP request: %w", err)
}
if err := c.auth.AddAuth(r); err != nil {
return nil, fmt.Errorf("adding authentication information to HTTP request: %w", err)
}

resp, err := c.client.Do(r)
if err != nil {
return nil, fmt.Errorf("executing HTTP request: %w", err)
}
return resp, nil
}

// unmarshalResponse unmarshalls a fleet-manager response. It returns an error if
// fleet-manager returns errors from its API.
// If the value v is nil the response is not marshalled into a struct, instead only checked for an API error.
func (c *Client) unmarshalResponse(resp *http.Response, v interface{}) error {
defer func() { _ = resp.Body.Close() }()
data, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("reading HTTP response body: %w", err)
}
if len(data) == 0 {
return nil
}

into := struct {
Kind string `json:"kind"`
}{}
err = json.Unmarshal(data, &into)
if err != nil {
return fmt.Errorf("extracting kind information from HTTP response: %w", err)
}

// Unmarshal error
if into.Kind == "Error" || into.Kind == "error" {
apiError := compat.Error{}
err = json.Unmarshal(data, &apiError)
if err != nil {
return fmt.Errorf("unmarshalling HTTP response as error: %w", err)
}
return errors.Errorf("API error (HTTP status %d) occured %s: %s", resp.StatusCode, apiError.Code, apiError.Reason)
}

if v == nil {
return nil
}

err = json.Unmarshal(data, v)
if err != nil {
return fmt.Errorf("unmarshalling HTTP response as %T: %w", v, err)
}

return nil
}

// NewAdminClient ...
func NewAdminClient(uriBase string, auth fleetmanager.Auth) (*Client, error) {
return &Client{
client: http.Client{},
auth: auth,
adminAPIEndpoint: fmt.Sprintf("%s/%s", uriBase, "api/rhacs/v1/admin"),
}, nil
}

// CreateCentral creates a central from the public fleet-manager API
func (c *Client) CreateCentral(request public.CentralRequestPayload) (*public.CentralRequest, error) {
reqBody, err := json.Marshal(request)
if err != nil {
return nil, fmt.Errorf("marshalling HTTP request: %w", err)
}

resp, err := c.newRequest(http.MethodPost, fmt.Sprintf("%s/dinosaurs?async=true", c.adminAPIEndpoint), bytes.NewBuffer(reqBody))
if err != nil {
return nil, fmt.Errorf("executing HTTP request: %w", err)
}

result := &public.CentralRequest{}
err = c.unmarshalResponse(resp, result)
if err != nil {
return nil, fmt.Errorf("unmarshalling HTTP response: %w", err)
}
return result, nil
}
143 changes: 140 additions & 3 deletions e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package e2e

import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"os"
"strings"
Expand All @@ -11,9 +13,11 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
openshiftRouteV1 "github.com/openshift/api/route/v1"
"github.com/stackrox/acs-fleet-manager/e2e/envtokenauth"
"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/fleetmanager"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/public"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/converters"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/services"
"github.com/stackrox/rox/operator/apis/platform/v1alpha1"
v1 "k8s.io/api/core/v1"
Expand All @@ -22,9 +26,17 @@ import (
)

// TODO(ROX-11468): Why is a central always created as a "eval" instance type?
var (
centralName = fmt.Sprintf("%s-%d", "e2e-test-central", time.Now().UnixMilli())
)
func newCentralName() string {
rnd := make([]byte, 8)
_, err := rand.Read(rnd)

if err != nil {
panic(fmt.Errorf("reading random bytes for unique central name: %w", err))
}
rndString := hex.EncodeToString(rnd)

return fmt.Sprintf("%s-%s", "e2e", rndString)
}

const (
defaultPolling = 1 * time.Second
Expand All @@ -35,26 +47,37 @@ const (
// TODO(ROX-11465): Use correct OCM_TOKEN for different clients (console.redhat.com, fleetshard)
var _ = Describe("Central", func() {
var client *fleetmanager.Client
var adminClient *Client

BeforeEach(func() {
authType := "OCM"
if val := os.Getenv("AUTH_TYPE"); val != "" {
authType = val
}
GinkgoWriter.Printf("AUTH_TYPE=%q\n", authType)

fleetManagerEndpoint := "http://localhost:8000"
if fmEndpointEnv := os.Getenv("FLEET_MANAGER_ENDPOINT"); fmEndpointEnv != "" {
fleetManagerEndpoint = fmEndpointEnv
}
GinkgoWriter.Printf("FLEET_MANAGER_ENDPOINT=%q\n", fleetManagerEndpoint)

auth, err := fleetmanager.NewAuth(authType)
Expect(err).ToNot(HaveOccurred())
client, err = fleetmanager.NewClient(fleetManagerEndpoint, "cluster-id", auth)
Expect(err).ToNot(HaveOccurred())

adminAuth, err := envtokenauth.CreateAuth("STATIC_TOKEN_ADMIN")
Expect(err).ToNot(HaveOccurred())
adminClient, err = NewAdminClient(fleetManagerEndpoint, adminAuth)
Expect(err).ToNot(HaveOccurred())

})

Describe("should be created and deployed to k8s", func() {
var err error

centralName := newCentralName()
request := public.CentralRequestPayload{
Name: centralName,
MultiAz: true,
Expand Down Expand Up @@ -220,6 +243,120 @@ var _ = Describe("Central", func() {
}).WithTimeout(waitTimeout).WithPolling(defaultPolling).Should(Equal(0))
})
})

Describe("should be created and deployed to k8s with admin API", func() {
var err error
centralName := newCentralName()

centralResources := public.ResourceRequirements{
Requests: public.ResourceList{
Cpu: "501m", Memory: "201M",
},
Limits: public.ResourceList{
Cpu: "502m", Memory: "202M",
},
}
centralSpec := public.CentralSpec{
Resources: centralResources,
}
scannerResources := public.ResourceRequirements{
Requests: public.ResourceList{
Cpu: "301m", Memory: "151M",
},
Limits: public.ResourceList{
Cpu: "302m", Memory: "152M",
},
}
scannerScaling := public.ScannerSpecAnalyzerScaling{
AutoScaling: "Enabled",
Replicas: 1,
MinReplicas: 1,
MaxReplicas: 2,
}
scannerSpec := public.ScannerSpec{
Analyzer: public.ScannerSpecAnalyzer{
Resources: scannerResources,
Scaling: scannerScaling,
},
}
request := public.CentralRequestPayload{
Name: centralName,
MultiAz: true,
CloudProvider: dpCloudProvider,
Region: dpRegion,
Central: centralSpec,
Scanner: scannerSpec,
}

var createdCentral *public.CentralRequest
var namespaceName string
It("should create central with custom resource configuration", func() {
createdCentral, err = adminClient.CreateCentral(request)
Expect(err).To(BeNil())
namespaceName, err = services.FormatNamespace(createdCentral.Id)
Expect(err).To(BeNil())
Expect(constants.DinosaurRequestStatusAccepted.String()).To(Equal(createdCentral.Status))
})

central := &v1alpha1.Central{}
It("should create central in its namespace on a managed cluster", func() {
Eventually(func() error {
return k8sClient.Get(context.Background(), ctrlClient.ObjectKey{Name: centralName, Namespace: namespaceName}, central)
}).WithTimeout(waitTimeout).WithPolling(defaultPolling).Should(Succeed())
})

It("central resources match configured settings", func() {
coreV1Resources := central.Spec.Central.DeploymentSpec.Resources
expectedResources, err := converters.ConvertPublicResourceRequirementsToCoreV1(&centralResources)
Expect(err).ToNot(HaveOccurred())
Expect(*coreV1Resources).To(Equal(expectedResources))
})

It("scanner analyzer resources match configured settings", func() {
coreV1Resources := central.Spec.Scanner.Analyzer.DeploymentSpec.Resources
expectedResources, err := converters.ConvertPublicResourceRequirementsToCoreV1(&scannerResources)
Expect(err).ToNot(HaveOccurred())
Expect(*coreV1Resources).To(Equal(expectedResources))

scaling := central.Spec.Scanner.Analyzer.Scaling
expectedScaling, err := converters.ConvertPublicScalingToV1(&scannerScaling)
Expect(err).ToNot(HaveOccurred())
Expect(*scaling).To(Equal(expectedScaling))
})

It("should transition central's state to ready", func() {
Eventually(func() string {
return centralStatus(createdCentral, client)
}).WithTimeout(waitTimeout).WithPolling(defaultPolling).Should(Equal(constants.DinosaurRequestStatusReady.String()))
})

It("should transition central to deprovisioning state", func() {
err = client.DeleteCentral(createdCentral.Id)
Expect(err).To(Succeed())
Eventually(func() string {
deprovisioningCentral, err := client.GetCentral(createdCentral.Id)
Expect(err).To(BeNil())
return deprovisioningCentral.Status
}).WithTimeout(waitTimeout).WithPolling(defaultPolling).Should(Equal(constants.DinosaurRequestStatusDeprovision.String()))
})

It("should delete central CR", func() {
Eventually(func() bool {
central := &v1alpha1.Central{}
err := k8sClient.Get(context.Background(), ctrlClient.ObjectKey{Name: centralName, Namespace: centralName}, central)
return apiErrors.IsNotFound(err)
}).WithTimeout(waitTimeout).WithPolling(defaultPolling).Should(BeTrue())
})

It("should remove central namespace", func() {
Eventually(func() bool {
ns := &v1.Namespace{}
err := k8sClient.Get(context.Background(), ctrlClient.ObjectKey{Name: namespaceName}, ns)
return apiErrors.IsNotFound(err)
}).WithTimeout(waitTimeout).WithPolling(defaultPolling).Should(BeTrue())
})

})
})

func getCentral(createdCentral *public.CentralRequest, client *fleetmanager.Client) *public.CentralRequest {
Expand Down
Loading

0 comments on commit 4f29bb2

Please sign in to comment.