Skip to content

Commit

Permalink
Actual providers implementation
Browse files Browse the repository at this point in the history
This defines the provider implementations in such a way that we
can instantiate them and check if a rule type is able to accomodate for those.

This is done through a new `ProviderBuidler` implementation which is in
charge of instantiating the provider type that's needed at a given time.

Note that we still use the GitHub provider for HTTP calls since it has a lot
of handy features (such as rate limiting handling) that we could leverage.
  • Loading branch information
JAORMX committed Sep 26, 2023
1 parent e2a22fe commit 3571999
Show file tree
Hide file tree
Showing 34 changed files with 989 additions and 424 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,4 @@ dbschema: ## generate database schema with schema spy, monitor file until doc is

mock:
mockgen -package mockdb -destination database/mock/store.go github.com/stacklok/mediator/internal/db Store
mockgen -package mockgh -destination internal/providers/github/mock/github.go -source internal/providers/github/github.go RestAPI
mockgen -package mockgh -destination internal/providers/github/mock/github.go -source pkg/providers/v1/providers.go GitHub
37 changes: 15 additions & 22 deletions cmd/dev/app/rule_type/rule_type.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ import (
"google.golang.org/protobuf/reflect/protoreflect"

"github.com/stacklok/mediator/cmd/dev/app"
"github.com/stacklok/mediator/internal/db"
"github.com/stacklok/mediator/internal/engine"
"github.com/stacklok/mediator/internal/engine/eval/rego"
"github.com/stacklok/mediator/internal/entities"
ghclient "github.com/stacklok/mediator/internal/providers/github"
"github.com/stacklok/mediator/internal/providers"
"github.com/stacklok/mediator/internal/util"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
)
Expand Down Expand Up @@ -105,12 +106,19 @@ func testCmdRun(cmd *cobra.Command, _ []string) error {
return fmt.Errorf("error getting relevant fragment: %w", err)
}

client, err := getProviderClient(context.Background(), rt, token)
if err != nil {
return fmt.Errorf("error getting provider client: %w", err)
}

eng, err := engine.NewRuleTypeEngine(rt, client, token)
// TODO: Read this from a providers file instead so we can make it pluggable
eng, err := engine.NewRuleTypeEngine(rt, providers.NewProviderBuilder(
&db.Provider{
Name: "test",
Implements: []db.ProviderType{
"rest",
"git",
"github",
},
},
db.ProviderAccessToken{},
token,
))
if err != nil {
return fmt.Errorf("error creating rule type engine: %w", err)
}
Expand Down Expand Up @@ -201,18 +209,3 @@ func readEntityFromFile(fpath string, entType pb.Entity) (protoreflect.ProtoMess

return out, nil
}

// getProviderClient returns a client for the provider specified in the rule type
// definition.
// TODO: This should be moved to a provider package and we should have some
// generic interface for clients.
func getProviderClient(ctx context.Context, rt *pb.RuleType, token string) (ghclient.RestAPI, error) {
switch rt.Context.Provider {
case ghclient.Github:
return ghclient.NewRestClient(ctx, ghclient.GitHubConfig{
Token: token,
}, "")
default:
return nil, fmt.Errorf("unknown provider: %s", rt.Context.Provider)
}
}
2 changes: 1 addition & 1 deletion database/migrations/000001_init.up.sql
Original file line number Diff line number Diff line change
Expand Up @@ -350,4 +350,4 @@ INSERT INTO user_roles (user_id, role_id) VALUES (1, 1);

-- Create default GitHub provider
INSERT INTO providers (name, group_id, implements, definition)
VALUES ('github', 1, ARRAY ['github', 'git', 'rest']::provider_type[], '{}');
VALUES ('github', 1, ARRAY ['github', 'git', 'rest']::provider_type[], '{"github": {}}');
4 changes: 2 additions & 2 deletions internal/container/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"

ghclient "github.com/stacklok/mediator/internal/providers/github"
"github.com/stacklok/mediator/internal/util"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
provifv1 "github.com/stacklok/mediator/pkg/providers/v1"
)

// REGISTRY is the default registry
Expand Down Expand Up @@ -80,7 +80,7 @@ var (
// GetArtifactSignatureAndWorkflowInfo returns the signature and workflow information as raw JSON for a given artifact
func GetArtifactSignatureAndWorkflowInfo(
ctx context.Context,
cli ghclient.RestAPI,
cli provifv1.Provider,
ownerLogin, artifactName, versionName string,
) (sigInfo json.RawMessage, workflowInfo json.RawMessage, err error) {
imageRef := artifactImageRef("", ownerLogin, artifactName, versionName)
Expand Down
44 changes: 34 additions & 10 deletions internal/controlplane/handlers_githubwebhooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ import (
"github.com/stacklok/mediator/internal/db"
"github.com/stacklok/mediator/internal/engine"
"github.com/stacklok/mediator/internal/providers"
ghclient "github.com/stacklok/mediator/internal/providers/github"
"github.com/stacklok/mediator/internal/util"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
provifv1 "github.com/stacklok/mediator/pkg/providers/v1"
)

// CONTAINER_TYPE is the type for container artifacts
Expand Down Expand Up @@ -134,7 +134,7 @@ func (s *Server) HandleGitHubWebHook() http.HandlerFunc {
// TODO: extract sender and event time from payload portably
m := message.NewMessage(uuid.New().String(), nil)
m.Metadata.Set("id", github.DeliveryID(r))
m.Metadata.Set("provider", ghclient.Github)
m.Metadata.Set("provider", string(db.ProviderTypeGithub))
m.Metadata.Set("source", "https://api.github.com/") // TODO: handle other sources

m.Metadata.Set("type", github.WebHookType(r))
Expand Down Expand Up @@ -374,11 +374,23 @@ func (s *Server) parseArtifactPublishedEvent(
return fmt.Errorf("error getting provider: %w", err)
}

cli, err := providers.BuildClient(ctx, dbrepo.Provider, g, s.store, s.cryptoEngine)
p, err := providers.GetProviderBuilder(ctx, prov, g, s.store, s.cryptoEngine)
if err != nil {
return fmt.Errorf("error building client: %w", err)
}

// NOTE(jaosorior): this webhook is very specific to github
if !p.Implements(db.ProviderTypeGithub) {
log.Printf("provider %s is not supported for github webhook", p.GetName())
return nil
}

cli, err := p.GetGitHub(ctx)
if err != nil {
log.Printf("error creating github provider: %v", err)
return nil
}

versionedArtifact, err := gatherVersionedArtifact(ctx, cli, s.store, whPayload)
if err != nil {
return fmt.Errorf("error gathering versioned artifact: %w", err)
Expand Down Expand Up @@ -419,11 +431,23 @@ func (s *Server) parsePullRequestModEvent(
return fmt.Errorf("error getting provider: %w", err)
}

cli, err := providers.BuildClient(ctx, prov.Name, g, s.store, s.cryptoEngine)
p, err := providers.GetProviderBuilder(ctx, prov, g, s.store, s.cryptoEngine)
if err != nil {
return fmt.Errorf("error building client: %w", err)
}

// NOTE(jaosorior): this webhook is very specific to github
if !p.Implements(db.ProviderTypeGithub) {
log.Printf("provider %s is not supported for github webhook", p.GetName())
return nil
}

cli, err := p.GetGitHub(ctx)
if err != nil {
log.Printf("error creating github provider: %v", err)
return nil
}

prEvalInfo, err := getPullRequestInfoFromPayload(ctx, whPayload)
if err != nil {
return fmt.Errorf("error getting pull request information from payload: %w", err)
Expand Down Expand Up @@ -502,7 +526,7 @@ func extractArtifactVersionFromPayload(ctx context.Context, payload map[string]a

func gatherArtifactInfo(
ctx context.Context,
client ghclient.RestAPI,
client provifv1.GitHub,
payload map[string]any,
) (*pb.Artifact, error) {
artifact, err := extractArtifactFromPayload(ctx, payload)
Expand Down Expand Up @@ -561,7 +585,7 @@ func lookUpVersionBySignature(

func gatherArtifactVersionInfo(
ctx context.Context,
cli ghclient.RestAPI,
cli provifv1.GitHub,
payload map[string]any,
artifactOwnerLogin, artifactName string,
) (*pb.ArtifactVersion, error) {
Expand All @@ -582,7 +606,7 @@ func gatherArtifactVersionInfo(

func gatherVersionedArtifact(
ctx context.Context,
cli ghclient.RestAPI,
cli provifv1.GitHub,
store db.Store,
payload map[string]any,
) (*pb.VersionedArtifact, error) {
Expand Down Expand Up @@ -623,7 +647,7 @@ func gatherVersionedArtifact(

func storeSignatureAndWorkflowInVersion(
ctx context.Context,
client ghclient.RestAPI,
client provifv1.GitHub,
artifactOwnerLogin, artifactName, packageVersionName string,
version *pb.ArtifactVersion,
) error {
Expand Down Expand Up @@ -651,7 +675,7 @@ func storeSignatureAndWorkflowInVersion(

func updateArtifactVersionFromRegistry(
ctx context.Context,
client ghclient.RestAPI,
client provifv1.GitHub,
payload map[string]any,
artifactOwnerLogin, artifactName string,
version *pb.ArtifactVersion,
Expand Down Expand Up @@ -827,7 +851,7 @@ func getPullRequestInfoFromPayload(

func updatePullRequestInfoFromProvider(
ctx context.Context,
cli ghclient.RestAPI,
cli provifv1.GitHub,
dbrepo db.Repository,
prEvalInfo *pb.PullRequest,
) error {
Expand Down
3 changes: 1 addition & 2 deletions internal/controlplane/handlers_organization.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,7 @@ func (s *Server) CreateOrganization(ctx context.Context,
Name: github.Github,
GroupID: grp.GroupId,
Implements: github.Implements,
// TODO add actual definition
Definition: json.RawMessage("{}"),
Definition: json.RawMessage(`{"github": {}}`),
})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to create provider: %v", err)
Expand Down
5 changes: 2 additions & 3 deletions internal/controlplane/handlers_repositories.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
github "github.com/stacklok/mediator/internal/providers/github"
"github.com/stacklok/mediator/internal/reconcilers"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
provifv1 "github.com/stacklok/mediator/pkg/providers/v1"
)

// RegisterRepository adds repositories to the database and registers a webhook
Expand Down Expand Up @@ -405,9 +406,7 @@ func (s *Server) SyncRepositories(ctx context.Context, in *pb.SyncRepositoriesRe
}

// Populate the database with the repositories using the GraphQL API
client, err := github.NewRestClient(ctx, github.GitHubConfig{
Token: token.AccessToken,
}, owner_filter)
client, err := github.NewRestClient(ctx, &provifv1.GitHubConfig{}, token.AccessToken, owner_filter)
if err != nil {
return nil, status.Errorf(codes.Internal, "cannot create github client: %v", err)
}
Expand Down
4 changes: 2 additions & 2 deletions internal/engine/eval/eval.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ import (
"github.com/stacklok/mediator/internal/engine/eval/rego"
"github.com/stacklok/mediator/internal/engine/eval/vulncheck"
engif "github.com/stacklok/mediator/internal/engine/interfaces"
ghclient "github.com/stacklok/mediator/internal/providers/github"
"github.com/stacklok/mediator/internal/providers"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
)

// NewRuleEvaluator creates a new rule data evaluator
func NewRuleEvaluator(rt *pb.RuleType, cli ghclient.RestAPI) (engif.Evaluator, error) {
func NewRuleEvaluator(rt *pb.RuleType, cli *providers.ProviderBuilder) (engif.Evaluator, error) {
e := rt.Def.GetEval()
if e == nil {
return nil, fmt.Errorf("rule type missing eval configuration")
Expand Down
4 changes: 2 additions & 2 deletions internal/engine/eval/vulncheck/actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ import (

"github.com/google/go-github/v53/github"

ghclient "github.com/stacklok/mediator/internal/providers/github"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
provifv1 "github.com/stacklok/mediator/pkg/providers/v1"
)

type prStatusHandler interface {
Expand All @@ -38,7 +38,7 @@ func newPrStatusHandler(
ctx context.Context,
action action,
pr *pb.PullRequest,
client ghclient.RestAPI,
client provifv1.GitHub,
) (prStatusHandler, error) {
switch action {
case actionReviewPr:
Expand Down
10 changes: 5 additions & 5 deletions internal/engine/eval/vulncheck/review.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import (
"github.com/google/go-github/v53/github"
"github.com/rs/zerolog"

ghclient "github.com/stacklok/mediator/internal/providers/github"
pb "github.com/stacklok/mediator/pkg/generated/protobuf/go/mediator/v1"
provifv1 "github.com/stacklok/mediator/pkg/providers/v1"
)

const (
Expand Down Expand Up @@ -100,7 +100,7 @@ func countLeadingWhitespace(line string) int {

func locateDepInPr(
_ context.Context,
client ghclient.RestAPI,
client provifv1.GitHub,
dep *pb.PrDependencies_ContextualDependency,
) (*reviewLocation, error) {
req, err := client.NewRequest("GET", dep.File.PatchUrl, nil)
Expand Down Expand Up @@ -138,7 +138,7 @@ func reviewBodyWithSuggestion(comment string) string {
}

type reviewPrHandler struct {
cli ghclient.RestAPI
cli provifv1.GitHub
pr *pb.PullRequest

mediatorReview *github.PullRequestReview
Expand All @@ -163,7 +163,7 @@ func withVulnsFoundReviewStatus(status *string) reviewPrHandlerOption {
func newReviewPrHandler(
ctx context.Context,
pr *pb.PullRequest,
cli ghclient.RestAPI,
cli provifv1.GitHub,
opts ...reviewPrHandlerOption,
) (*reviewPrHandler, error) {
if pr == nil {
Expand Down Expand Up @@ -352,7 +352,7 @@ type commitStatusPrHandler struct {
func newCommitStatusPrHandler(
ctx context.Context,
pr *pb.PullRequest,
client ghclient.RestAPI,
client provifv1.GitHub,
) (prStatusHandler, error) {
// create a reviewPrHandler and embed it in the commitStatusPrHandler
rph, err := newReviewPrHandler(
Expand Down
10 changes: 5 additions & 5 deletions internal/engine/eval/vulncheck/review_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func TestReviewPrHandlerNoVulnerabilities(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockClient := mock_ghclient.NewMockRestAPI(ctrl)
mockClient := mock_ghclient.NewMockGitHub(ctrl)
pr := &pb.PullRequest{
Url: "https://api.github.com/repos/jakubtestorg/bad-npm/pulls/43",
CommitSha: commitSHA,
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestReviewPrHandlerVulnerabilitiesDifferentIdentities(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockClient := mock_ghclient.NewMockRestAPI(ctrl)
mockClient := mock_ghclient.NewMockGitHub(ctrl)
pr := &pb.PullRequest{
Url: "https://api.github.com/repos/jakubtestorg/bad-npm/pulls/43",
CommitSha: commitSHA,
Expand Down Expand Up @@ -171,7 +171,7 @@ func TestReviewPrHandlerVulnerabilitiesDismissReview(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockClient := mock_ghclient.NewMockRestAPI(ctrl)
mockClient := mock_ghclient.NewMockGitHub(ctrl)
pr := &pb.PullRequest{
Url: "https://api.github.com/repos/jakubtestorg/bad-npm/pulls/43",
CommitSha: commitSHA,
Expand Down Expand Up @@ -222,7 +222,7 @@ func TestCommitStatusHandlerNoVulnerabilities(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockClient := mock_ghclient.NewMockRestAPI(ctrl)
mockClient := mock_ghclient.NewMockGitHub(ctrl)
pr := &pb.PullRequest{
Url: "https://api.github.com/repos/jakubtestorg/bad-npm/pulls/43",
CommitSha: commitSHA,
Expand Down Expand Up @@ -269,7 +269,7 @@ func TestCommitStatusPrHandlerWithVulnerabilities(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

mockClient := mock_ghclient.NewMockRestAPI(ctrl)
mockClient := mock_ghclient.NewMockGitHub(ctrl)
pr := &pb.PullRequest{
Url: "https://api.github.com/repos/jakubtestorg/bad-npm/pulls/43",
CommitSha: commitSHA,
Expand Down
Loading

0 comments on commit 3571999

Please sign in to comment.