feat: use Keto 0.11 for authorization

Dockerfile

@@ -17,7 +17,7 @@ COPY api api/
 COPY go.mod .
 COPY go.sum .
 COPY db-migrations ./db-migrations
-RUN go build -o bin/mlp-api ./api/cmd/main.go
+RUN go build -o bin/mlp-api ./api/main.go
 
 # ============================================================
 # Build stage 3: Run the app
@@ -28,3 +28,4 @@ COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp
 COPY --from=go-builder /src/api/db-migrations ./db-migrations
 
 ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
+CMD ["serve"]

Makefile

@@ -16,7 +16,7 @@ all: setup init-dep lint test clean build run
 setup:
 	@echo "> Setting up tools..."
 	@test -x $(shell go env GOPATH)/bin/golangci-lint || \
-		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.48.0/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.53.3/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.53.3
 
 .PHONY: init-dep
 init-dep: init-dep-ui init-dep-api
@@ -86,7 +86,7 @@ build-ui: clean-ui
 .PHONY: build-api
 build-api: clean-bin
 	@echo "> Building API binary ..."
-	@cd ${API_PATH} && go build -o ../bin/${BIN_NAME} ./cmd/main.go
+	@cd ${API_PATH} && go build -o ../bin/${BIN_NAME} main.go
 
 .PHONY: build-api-image
 build-api-image: version
@@ -111,7 +111,7 @@ build-image: version
 .PHONY: run
 run: local-env
 	@echo "> Running application ..."
-	@go run api/cmd/main.go --config config-dev.yaml
+	@go run api/main.go serve --config config-dev.yaml
 
 .PHONY: start-ui
 start-ui:

api.Dockerfile

@@ -10,12 +10,13 @@ COPY api api/
 COPY go.mod .
 COPY go.sum .
 
-RUN go build -o bin/mlp-api ./api/cmd/main.go
+RUN go build -o bin/mlp-api ./api/main.go
 
 # Clean image with mlp-api binary
 FROM alpine:3.16
 
 COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp
 COPY db-migrations ./db-migrations
 
-ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
+ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
+CMD ["serve"]

api/api/api_test.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -75,12 +76,12 @@ func (s *APITestSuite) SetupTest() {
 	s.Require().NoError(err, "Failed to create app context")
 
 	// create project and otherProject
-	s.mainProject, err = appCtx.ProjectsService.CreateProject(&models.Project{
+	s.mainProject, err = appCtx.ProjectsService.CreateProject(context.Background(), &models.Project{
 		Name: "test-project",
 	})
 	s.Require().NoError(err, "Failed to create project")
 
-	s.otherProject, err = appCtx.ProjectsService.CreateProject(&models.Project{
+	s.otherProject, err = appCtx.ProjectsService.CreateProject(context.Background(), &models.Project{
 		Name: "other-project",
 	})
 	s.Require().NoError(err, "Failed to create other project")

api/api/projects_api.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/caraml-dev/mlp/api/log"
 	"github.com/caraml-dev/mlp/api/models"
-	"github.com/caraml-dev/mlp/api/pkg/authz/enforcer"
 	apperror "github.com/caraml-dev/mlp/api/pkg/errors"
 )
 
@@ -17,14 +16,12 @@ type ProjectsController struct {
 }
 
 func (c *ProjectsController) ListProjects(r *http.Request, vars map[string]string, _ interface{}) *Response {
-	projects, err := c.ProjectsService.ListProjects(vars["name"])
+	projects, err := c.ProjectsService.ListProjects(r.Context(), vars["name"], vars["user"])
 	if err != nil {
 		log.Errorf("error fetching projects: %s", err)
 		return FromError(err)
 	}
 
-	user := vars["user"]
-	projects, err = c.filterAuthorizedProjects(user, projects, enforcer.ActionRead)
 	if err != nil {
 		return InternalServerError(err.Error())
 	}
@@ -57,7 +54,7 @@ func (c *ProjectsController) CreateProject(r *http.Request, vars map[string]stri
 
 	user := vars["user"]
 	project.Administrators = addRequester(user, project.Administrators)
-	project, err = c.ProjectsService.CreateProject(project)
+	project, err = c.ProjectsService.CreateProject(r.Context(), project)
 	if err != nil {
 		log.Errorf("error creating project %s: %s", project.Name, err)
 		return FromError(err)
@@ -85,7 +82,7 @@ func (c *ProjectsController) UpdateProject(r *http.Request, vars map[string]stri
 	project.Team = newProject.Team
 	project.Stream = newProject.Stream
 	project.Labels = newProject.Labels
-	project, err = c.ProjectsService.UpdateProject(project)
+	project, err = c.ProjectsService.UpdateProject(r.Context(), project)
 	if err != nil {
 		log.Errorf("error updating project %s: %s", project.Name, err)
 		return FromError(err)
@@ -105,36 +102,6 @@ func (c *ProjectsController) GetProject(r *http.Request, vars map[string]string,
 	return Ok(project)
 }
 
-func (c *ProjectsController) filterAuthorizedProjects(
-	user string,
-	projects []*models.Project,
-	action string,
-) ([]*models.Project, error) {
-	if c.AuthorizationEnabled {
-		projectIds := make([]string, 0)
-		allowedProjects := make([]*models.Project, 0)
-		projectMap := make(map[string]*models.Project)
-		for _, project := range projects {
-			projectID := fmt.Sprintf("projects:%s", project.ID)
-			projectIds = append(projectIds, projectID)
-			projectMap[projectID] = project
-		}
-
-		allowedProjectIds, err := c.Enforcer.FilterAuthorizedResource(user, projectIds, action)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, projectID := range allowedProjectIds {
-			allowedProjects = append(allowedProjects, projectMap[projectID])
-		}
-
-		return allowedProjects, nil
-	}
-
-	return projects, nil
-}
-
 func (c *ProjectsController) Routes() []Route {
 	return []Route{
 		{

api/api/router.go

@@ -33,14 +33,16 @@ type AppContext struct {
 	SecretStorageService service.SecretStorageService
 	DefaultSecretStorage *models.SecretStorage
 
-	AuthorizationEnabled bool
-	Enforcer             enforcer.Enforcer
+	AuthorizationEnabled       bool
+	UseAuthorizationMiddleware bool
+	Enforcer                   enforcer.Enforcer
 }
 
 func NewAppContext(db *gorm.DB, cfg *config.Config) (ctx *AppContext, err error) {
 	var authEnforcer enforcer.Enforcer
 	if cfg.Authorization.Enabled {
-		enforcerCfg := enforcer.NewEnforcerBuilder().URL(cfg.Authorization.KetoServerURL).Product("mlp")
+		enforcerCfg := enforcer.NewEnforcerBuilder()
+		enforcerCfg.KetoEndpoints(cfg.Authorization.KetoRemoteRead, cfg.Authorization.KetoRemoteWrite)
 		if cfg.Authorization.Caching.Enabled {
 			enforcerCfg = enforcerCfg.WithCaching(
 				cfg.Authorization.Caching.KeyExpirySeconds,
@@ -94,13 +96,14 @@ func NewAppContext(db *gorm.DB, cfg *config.Config) (ctx *AppContext, err error)
 		projectRepository, storageClientRegistry, defaultSecretStorage)
 
 	return &AppContext{
-		ApplicationService:   applicationService,
-		ProjectsService:      projectsService,
-		SecretService:        secretService,
-		SecretStorageService: secretStorageService,
-		AuthorizationEnabled: cfg.Authorization.Enabled,
-		Enforcer:             authEnforcer,
-		DefaultSecretStorage: defaultSecretStorage,
+		ApplicationService:         applicationService,
+		ProjectsService:            projectsService,
+		SecretService:              secretService,
+		SecretStorageService:       secretStorageService,
+		AuthorizationEnabled:       cfg.Authorization.Enabled,
+		UseAuthorizationMiddleware: cfg.Authorization.UseMiddleware,
+		Enforcer:                   authEnforcer,
+		DefaultSecretStorage:       defaultSecretStorage,
 	}, nil
 }
 
@@ -180,7 +183,7 @@ func NewRouter(appCtx *AppContext, controllers []Controller) *mux.Router {
 	router := mux.NewRouter().StrictSlash(true)
 	validator := validation.NewValidator()
 
-	if appCtx.AuthorizationEnabled {
+	if appCtx.AuthorizationEnabled && appCtx.UseAuthorizationMiddleware {
 		authzMiddleware := middleware.NewAuthorizer(appCtx.Enforcer)
 		router.Use(authzMiddleware.AuthorizationMiddleware)
 	}

api/cmd/bootstrap.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"context"
+
+	"github.com/knadh/koanf"
+	"github.com/knadh/koanf/parsers/yaml"
+	"github.com/knadh/koanf/providers/file"
+
+	"github.com/spf13/cobra"
+
+	"github.com/caraml-dev/mlp/api/config"
+	"github.com/caraml-dev/mlp/api/log"
+	"github.com/caraml-dev/mlp/api/pkg/authz/enforcer"
+)
+
+type BootstrapRoleMembers struct {
+	ProjectReaders []string
+	MLPAdmins      []string
+}
+
+var (
+	bootstrapRoleMembersInputFile string
+	bootstrapCmd                  = &cobra.Command{
+		Use:   "bootstrap",
+		Short: "Start bootstrap job to populate Keto",
+		Run: func(cmd *cobra.Command, args []string) {
+			bootstrapRoleMembers, err := loadRoleMemberFromInputFile(bootstrapRoleMembersInputFile)
+			if err != nil {
+				log.Panicf("unable to load role members from input file: %v", err)
+			}
+			err = startKetoBootstrap(globalConfig, bootstrapRoleMembers)
+			if err != nil {
+				log.Panicf("unable to bootstrap keto: %v", err)
+			}
+		},
+	}
+)
+
+func init() {
+	bootstrapCmd.Flags().StringVarP(&bootstrapRoleMembersInputFile, "role-members", "r", "",
+		"Path to an input file that map roles to members")
+	err := bootstrapCmd.MarkFlagRequired("role-members")
+	if err != nil {
+		log.Panicf("unable to mark flag as required: %v", err)
+	}
+}
+
+func loadRoleMemberFromInputFile(path string) (*BootstrapRoleMembers, error) {
+	bootstrapRoleMembers := &BootstrapRoleMembers{
+		ProjectReaders: []string{},
+		MLPAdmins:      []string{},
+	}
+	k := koanf.New(".")
+	err := k.Load(file.Provider(path), yaml.Parser())
+	if err != nil {
+		return nil, err
+	}
+	err = k.Unmarshal("", bootstrapRoleMembers)
+	if err != nil {
+		return nil, err
+	}
+	return bootstrapRoleMembers, nil
+}
+
+func startKetoBootstrap(globalCfg *config.Config, bootstrapOpts *BootstrapRoleMembers) error {
+	authEnforcer, err := enforcer.NewEnforcerBuilder().
+		KetoEndpoints(globalCfg.Authorization.KetoRemoteRead, globalCfg.Authorization.KetoRemoteWrite).
+		Build()
+	if err != nil {
+		return err
+	}
+	updateRequest := enforcer.NewAuthorizationUpdateRequest()
+	updateRequest.SetRoleMembers(enforcer.MLPProjectsReaderRole, bootstrapOpts.ProjectReaders)
+	updateRequest.SetRoleMembers(enforcer.MLPAdminRole, bootstrapOpts.MLPAdmins)
+	err = authEnforcer.UpdateAuthorization(context.Background(), updateRequest)
+	if err != nil {
+		return err
+	}
+	return nil
+}

api/cmd/root.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/caraml-dev/mlp/api/config"
+	"github.com/caraml-dev/mlp/api/log"
+)
+
+var (
+	configFiles  []string
+	globalConfig *config.Config
+	rootCmd      = &cobra.Command{
+		Use:   "mlp",
+		Short: "CaraML Machine Learning Platform Console",
+		Long: "CaraML Machine Learning Platform Console, which provides a web UI to interact with different CaraML " +
+			"services. If no subcommand are provided, serve command will be run as default.",
+		// Run serve command by default if non is specified
+		Run: func(cmd *cobra.Command, args []string) {
+			serveCmd.Run(cmd, args)
+		},
+	}
+)
+
+func init() {
+	cobra.OnInitialize(initConfig)
+	rootCmd.PersistentFlags().StringSliceVarP(&configFiles, "config", "c", []string{},
+		"Comma separated list of config files to load. The last config file will take precedence over the "+
+			"previous ones.")
+	rootCmd.AddCommand(serveCmd)
+	rootCmd.AddCommand(bootstrapCmd)
+}
+
+func initConfig() {
+	var err error
+	globalConfig, err = config.LoadAndValidate(configFiles...)
+	if err != nil {
+		log.Fatalf("failed initializing config: %v", err)
+	}
+}
+
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		log.Fatalf("failed executing root command: %v", err)
+	}
+}

api/cmd/main.go → api/cmd/serve.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"encoding/json"
@@ -13,7 +13,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/heptiolabs/healthcheck"
 	"github.com/rs/cors"
-	flag "github.com/spf13/pflag"
+	"github.com/spf13/cobra"
 
 	"github.com/caraml-dev/mlp/api/api"
 	apiV2 "github.com/caraml-dev/mlp/api/api/v2"
@@ -23,15 +23,17 @@ import (
 	"github.com/caraml-dev/mlp/api/pkg/authz/enforcer"
 )
 
-func main() {
-	configFiles := flag.StringSliceP("config", "c", []string{}, "Path to a configuration files")
-	flag.Parse()
-
-	cfg, err := config.LoadAndValidate(*configFiles...)
-	if err != nil {
-		log.Panicf("failed initializing config: %v", err)
+var (
+	serveCmd = &cobra.Command{
+		Use:   "serve",
+		Short: "Start MLP API server",
+		Run: func(cmd *cobra.Command, args []string) {
+			startServer(globalConfig)
+		},
 	}
+)
 
+func startServer(cfg *config.Config) {
 	// init db
 	db, err := database.InitDB(cfg.Database)
 	if err != nil {