diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9f4036b2..34f8831a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -28,7 +28,7 @@ env: LOCAL_REGISTRY: registry.localhost:5000 CARAML_CHARTS_REPOSITORY: romanwozniak/helm-charts - CARAML_CHARTS_REF: feat-mlp-support-configmap + CARAML_CHARTS_REF: feat-mlp-apps-configuration jobs: build-ui: diff --git a/Makefile b/Makefile index 9281a402..5a870ec1 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,6 @@ UI_BUILD_PATH := ${UI_PATH}/build API_PATH := api API_ALL_PACKAGES := $(shell cd ${API_PATH} && go list ./... | grep -v github.com/gojek/mlp/client | grep -v mocks) BIN_NAME := $(if ${APP_NAME},${APP_NAME},mlp) -DOCKER_COMPOSE_BIN := "docker-compose" -DOCKER_COMPOSE_ARG := "" all: setup init-dep lint test clean build run @@ -150,22 +148,22 @@ generate-client: .PHONY: local-db local-db: @echo "> Starting up DB ..." - @$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) up -d postgres + @docker-compose up -d postgres .PHONY: start-keto start-keto: @echo "> Starting up keto server ..." - @$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) up -d keto + @docker-compose up -d keto .PHONY: stop-docker stop-docker: @echo "> Stopping Docker compose ..." - @$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) down + @docker-compose down .PHONY: swagger-ui swagger-ui: @echo "> Starting up Swagger UI ..." - @$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) up -d swagger-ui + @docker-compose up -d swagger-ui .PHONY: version version: diff --git a/api/api/applications_api.go b/api/api/applications_api.go index a846009e..6563eef5 100644 --- a/api/api/applications_api.go +++ b/api/api/applications_api.go @@ -1,16 +1,29 @@ package api -import "net/http" +import ( + http "net/http" +) type ApplicationsController struct { *AppContext } -func (c *ApplicationsController) ListApplications(r *http.Request, _ map[string]string, _ interface{}) *Response { +func (c *ApplicationsController) ListApplications(_ *http.Request, _ map[string]string, _ interface{}) *Response { applications, err := c.ApplicationService.List() if err != nil { return InternalServerError(err.Error()) } return Ok(applications) +} +func (c *ApplicationsController) Routes() []Route { + return []Route{ + { + http.MethodGet, + "/applications", + nil, + c.ListApplications, + "ListApplications", + }, + } } diff --git a/api/api/projects_api.go b/api/api/projects_api.go index 8c09cc75..72981a5b 100644 --- a/api/api/projects_api.go +++ b/api/api/projects_api.go @@ -5,11 +5,10 @@ import ( "net/http" "strings" - "github.com/gojek/mlp/api/pkg/authz/enforcer" - "github.com/jinzhu/gorm" - "github.com/gojek/mlp/api/log" "github.com/gojek/mlp/api/models" + "github.com/gojek/mlp/api/pkg/authz/enforcer" + "github.com/jinzhu/gorm" ) type ProjectsController struct { @@ -135,6 +134,39 @@ func (c *ProjectsController) filterAuthorizedProjects( return projects, nil } +func (c *ProjectsController) Routes() []Route { + return []Route{ + { + http.MethodGet, + "/projects/{project_id:[0-9]+}", + nil, + c.GetProject, + "GetProject", + }, + { + http.MethodGet, + "/projects", + nil, + c.ListProjects, + "ListProjects", + }, + { + http.MethodPost, + "/projects", + models.Project{}, + c.CreateProject, + "CreateProject", + }, + { + http.MethodPut, + "/projects/{project_id:[0-9]+}", + models.Project{}, + c.UpdateProject, + "UpdateProject", + }, + } +} + // addRequester add requester to users slice if it doesn't exists func addRequester(requester string, users []string) []string { for _, user := range users { diff --git a/api/api/projects_api_test.go b/api/api/projects_api_test.go index 4c50b1d6..cb818df4 100644 --- a/api/api/projects_api_test.go +++ b/api/api/projects_api_test.go @@ -17,7 +17,6 @@ import ( "github.com/gojek/mlp/api/it/database" "github.com/gojek/mlp/api/models" - enforcerMock "github.com/gojek/mlp/api/pkg/authz/enforcer/mocks" "github.com/gojek/mlp/api/service" "github.com/gojek/mlp/api/storage" ) @@ -207,13 +206,15 @@ func TestCreateProject(t *testing.T) { _, err := prjStorage.Save(tC.existingProject) assert.NoError(t, err) } - projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, &enforcerMock.Enforcer{}, false) + projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, nil, false) assert.NoError(t, err) - r := NewRouter(AppContext{ + appCtx := &AppContext{ ProjectsService: projectService, AuthorizationEnabled: false, - }) + } + controllers := []Controller{&ProjectsController{appCtx}} + r := NewRouter(appCtx, controllers) requestByte, _ := json.Marshal(tC.body) req, err := http.NewRequest(http.MethodPost, "/v1/projects", bytes.NewReader(requestByte)) @@ -315,13 +316,15 @@ func TestListProjects(t *testing.T) { assert.NoError(t, err) } } - projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, &enforcerMock.Enforcer{}, false) + projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, nil, false) assert.NoError(t, err) - r := NewRouter(AppContext{ + appCtx := &AppContext{ ProjectsService: projectService, AuthorizationEnabled: false, - }) + } + controllers := []Controller{&ProjectsController{appCtx}} + r := NewRouter(appCtx, controllers) req, err := http.NewRequest(http.MethodGet, "/v1/projects", nil) if err != nil { @@ -473,13 +476,15 @@ func TestUpdateProject(t *testing.T) { _, err := prjStorage.Save(tC.existingProject) assert.NoError(t, err) } - projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, &enforcerMock.Enforcer{}, false) + projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, nil, false) assert.NoError(t, err) - r := NewRouter(AppContext{ + appCtx := &AppContext{ ProjectsService: projectService, AuthorizationEnabled: false, - }) + } + controllers := []Controller{&ProjectsController{appCtx}} + r := NewRouter(appCtx, controllers) requestByte, _ := json.Marshal(tC.body) req, err := http.NewRequest(http.MethodPut, "/v1/projects/"+tC.projectID.String(), bytes.NewReader(requestByte)) @@ -590,13 +595,15 @@ func TestGetProject(t *testing.T) { _, err := prjStorage.Save(tC.existingProject) assert.NoError(t, err) } - projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, &enforcerMock.Enforcer{}, false) + projectService, err := service.NewProjectsService(mlflowTrackingURL, prjStorage, nil, false) assert.NoError(t, err) - r := NewRouter(AppContext{ + appCtx := &AppContext{ ProjectsService: projectService, AuthorizationEnabled: false, - }) + } + controllers := []Controller{&ProjectsController{appCtx}} + r := NewRouter(appCtx, controllers) req, err := http.NewRequest(http.MethodGet, "/v1/projects/"+tC.projectID.String(), nil) if err != nil { diff --git a/api/api/router.go b/api/api/router.go index 24b917a3..2eb09bcb 100644 --- a/api/api/router.go +++ b/api/api/router.go @@ -6,18 +6,22 @@ import ( "net/http" "reflect" - "github.com/gojek/mlp/api/pkg/instrumentation/newrelic" - "github.com/go-playground/validator" - "github.com/gojek/mlp/api/pkg/authz/enforcer" - "github.com/gorilla/mux" - + "github.com/gojek/mlp/api/config" "github.com/gojek/mlp/api/middleware" - "github.com/gojek/mlp/api/models" + "github.com/gojek/mlp/api/pkg/authz/enforcer" + "github.com/gojek/mlp/api/pkg/instrumentation/newrelic" "github.com/gojek/mlp/api/service" + "github.com/gojek/mlp/api/storage" "github.com/gojek/mlp/api/validation" + "github.com/gorilla/mux" + "github.com/jinzhu/gorm" ) +type Controller interface { + Routes() []Route +} + type AppContext struct { ApplicationService service.ApplicationService ProjectsService service.ProjectsService @@ -27,21 +31,60 @@ type AppContext struct { Enforcer enforcer.Enforcer } +func NewAppContext(db *gorm.DB, cfg *config.Config) (ctx *AppContext, err error) { + var authEnforcer enforcer.Enforcer + if cfg.Authorization.Enabled { + authEnforcer, err = enforcer.NewEnforcerBuilder(). + URL(cfg.Authorization.KetoServerURL). + Product("mlp"). + Build() + + if err != nil { + return nil, fmt.Errorf("failed to initialize authorization service: %v", err) + } + } + + applicationService, err := service.NewApplicationService(db) + if err != nil { + return nil, fmt.Errorf("failed to initialize applications service: %v", err) + } + + projectsService, err := service.NewProjectsService( + cfg.Mlflow.TrackingURL, + storage.NewProjectStorage(db), + authEnforcer, + cfg.Authorization.Enabled) + + if err != nil { + return nil, fmt.Errorf("failed to initialize projects service: %v", err) + } + + secretService := service.NewSecretService(storage.NewSecretStorage(db, cfg.EncryptionKey)) + + return &AppContext{ + ApplicationService: applicationService, + ProjectsService: projectsService, + SecretService: secretService, + AuthorizationEnabled: cfg.Authorization.Enabled, + Enforcer: authEnforcer, + }, nil +} + // type Handler func(r *http.Request, vars map[string]string, body interface{}) *Response type Handler func(r *http.Request, vars map[string]string, body interface{}) *Response type Route struct { - method string - path string - body interface{} - handler Handler - name string + Method string + Path string + Body interface{} + Handler Handler + Name string } func (route Route) HandlerFunc(validate *validator.Validate) http.HandlerFunc { var bodyType reflect.Type - if route.body != nil { - bodyType = reflect.TypeOf(route.body) + if route.Body != nil { + bodyType = reflect.TypeOf(route.Body) } return func(w http.ResponseWriter, r *http.Request) { @@ -66,113 +109,35 @@ func (route Route) HandlerFunc(validate *validator.Validate) http.HandlerFunc { return BadRequest(errMessage) } } - return route.handler(r, vars, body) + return route.Handler(r, vars, body) }() response.WriteTo(w) } } -func NewRouter(appCtx AppContext) *mux.Router { +func NewRouter(appCtx *AppContext, controllers []Controller) *mux.Router { + router := mux.NewRouter().StrictSlash(true) validator := validation.NewValidator() - applicationsController := ApplicationsController{&appCtx} - projectsController := ProjectsController{&appCtx} - secretController := SecretsController{&appCtx} - - routes := []Route{ - //Applications API - { - http.MethodGet, - "/applications", - nil, - applicationsController.ListApplications, - "ListApplications", - }, - - //Projects API - { - http.MethodGet, - "/projects/{project_id:[0-9]+}", - nil, - projectsController.GetProject, - "GetProject", - }, - { - http.MethodGet, - "/projects", - nil, - projectsController.ListProjects, - "ListProjects", - }, - { - http.MethodPost, - "/projects", - models.Project{}, - projectsController.CreateProject, - "CreateProject", - }, - { - http.MethodPut, - "/projects/{project_id:[0-9]+}", - models.Project{}, - projectsController.UpdateProject, - "UpdateProject", - }, - - // Secret Management API - { - http.MethodGet, - "/projects/{project_id:[0-9]+}/secrets", - nil, - secretController.ListSecret, - "ListSecret", - }, - { - http.MethodPost, - "/projects/{project_id:[0-9]+}/secrets", - models.Secret{}, - secretController.CreateSecret, - "CreateSecret", - }, - { - http.MethodPatch, - "/projects/{project_id:[0-9]+}/secrets/{secret_id}", - models.Secret{}, - secretController.UpdateSecret, - "UpdateSecret", - }, - { - http.MethodDelete, - "/projects/{project_id:[0-9]+}/secrets/{secret_id}", - nil, - secretController.DeleteSecret, - "DeleteSecret", - }, - } - - var authzMiddleware *middleware.Authorizer - var projCreationMiddleware *middleware.ProjectCreation if appCtx.AuthorizationEnabled { - authzMiddleware = middleware.NewAuthorizer(appCtx.Enforcer) + authzMiddleware := middleware.NewAuthorizer(appCtx.Enforcer) + router.Use(authzMiddleware.AuthorizationMiddleware) } - router := mux.NewRouter().StrictSlash(true) - for _, r := range routes { - _, handler := newrelic.WrapHandle(r.name, r.HandlerFunc(validator)) + for _, c := range controllers { + for _, r := range c.Routes() { + _, handler := newrelic.WrapHandle(r.Name, r.HandlerFunc(validator)) - if r.name == "CreateProject" { - handler = projCreationMiddleware.ProjectCreationMiddleware(handler) - } + if r.Name == "CreateProject" { + handler = middleware.ProjectCreationMiddleware(handler) + } - if appCtx.AuthorizationEnabled { - handler = authzMiddleware.AuthorizationMiddleware(handler) + router.Name(r.Name). + Methods(r.Method). + Path(r.Path). + Handler(handler) } - - router.Name(r.name). - Methods(r.method). - Path(r.path). - Handler(handler) } return router diff --git a/api/api/secrets_api.go b/api/api/secrets_api.go index 186f0ee4..40c366ac 100644 --- a/api/api/secrets_api.go +++ b/api/api/secrets_api.go @@ -100,3 +100,36 @@ func (c *SecretsController) ListSecret(r *http.Request, vars map[string]string, } return Ok(secrets) } + +func (c *SecretsController) Routes() []Route { + return []Route{ + { + http.MethodGet, + "/projects/{project_id:[0-9]+}/secrets", + nil, + c.ListSecret, + "ListSecret", + }, + { + http.MethodPost, + "/projects/{project_id:[0-9]+}/secrets", + models.Secret{}, + c.CreateSecret, + "CreateSecret", + }, + { + http.MethodPatch, + "/projects/{project_id:[0-9]+}/secrets/{secret_id}", + models.Secret{}, + c.UpdateSecret, + "UpdateSecret", + }, + { + http.MethodDelete, + "/projects/{project_id:[0-9]+}/secrets/{secret_id}", + nil, + c.DeleteSecret, + "DeleteSecret", + }, + } +} diff --git a/api/api/v2/applications_api.go b/api/api/v2/applications_api.go new file mode 100644 index 00000000..df3e9920 --- /dev/null +++ b/api/api/v2/applications_api.go @@ -0,0 +1,27 @@ +package api + +import ( + "net/http" + + "github.com/gojek/mlp/api/api" + "github.com/gojek/mlp/api/models/v2" +) + +type ApplicationsController struct { + Apps []models.Application +} + +func (c *ApplicationsController) ListApplications(_ *http.Request, _ map[string]string, _ interface{}) *api.Response { + return api.Ok(c.Apps) +} + +func (c *ApplicationsController) Routes() []api.Route { + return []api.Route{ + { + Method: http.MethodGet, + Path: "/applications", + Handler: c.ListApplications, + Name: "ListApplications", + }, + } +} diff --git a/api/cmd/main.go b/api/cmd/main.go index 911a563a..59912fa0 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -9,12 +9,10 @@ import ( "strings" "github.com/gojek/mlp/api/api" + apiV2 "github.com/gojek/mlp/api/api/v2" "github.com/gojek/mlp/api/config" "github.com/gojek/mlp/api/database" "github.com/gojek/mlp/api/log" - "github.com/gojek/mlp/api/pkg/authz/enforcer" - "github.com/gojek/mlp/api/service" - "github.com/gojek/mlp/api/storage" "github.com/gorilla/mux" "github.com/heptiolabs/healthcheck" "github.com/rs/cors" @@ -27,46 +25,36 @@ func main() { cfg, err := config.LoadAndValidate(*configFiles...) if err != nil { - log.Panicf("Failed initializing config: %v", err) + log.Panicf("failed initializing config: %v", err) } // init db db, err := database.InitDB(cfg.Database) if err != nil { - panic(err) + log.Panicf("unable to initialize DB connectivity: %v", err) } defer db.Close() - applicationService, _ := service.NewApplicationService(db) - authEnforcer, _ := enforcer.NewEnforcerBuilder(). - URL(cfg.Authorization.KetoServerURL). - Product("mlp"). - Build() - - projectsService, err := service.NewProjectsService( - cfg.Mlflow.TrackingURL, - storage.NewProjectStorage(db), - authEnforcer, - cfg.Authorization.Enabled) - + appCtx, err := api.NewAppContext(db, cfg) if err != nil { - log.Panicf("unable to initialize project service: %v", err) + log.Panicf("unable to initialize application context: %v", err) } - secretService := service.NewSecretService(storage.NewSecretStorage(db, cfg.EncryptionKey)) + router := mux.NewRouter() - appCtx := api.AppContext{ - ApplicationService: applicationService, - ProjectsService: projectsService, - SecretService: secretService, + mount(router, "/v1/internal", healthcheck.NewHandler()) - AuthorizationEnabled: cfg.Authorization.Enabled, - Enforcer: authEnforcer, + v1Controllers := []api.Controller{ + &api.ApplicationsController{AppContext: appCtx}, + &api.ProjectsController{AppContext: appCtx}, + &api.SecretsController{AppContext: appCtx}, } + mount(router, "/v1", api.NewRouter(appCtx, v1Controllers)) - router := mux.NewRouter() - mount(router, "/v1/internal", healthcheck.NewHandler()) - mount(router, "/v1", api.NewRouter(appCtx)) + v2Controllers := []api.Controller{ + &apiV2.ApplicationsController{Apps: cfg.Applications}, + } + mount(router, "/v2", api.NewRouter(appCtx, v2Controllers)) uiEnv := uiEnvHandler{ APIURL: cfg.APIHost, diff --git a/api/config/config.go b/api/config/config.go index d2a19d50..fa3bd081 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/go-playground/validator/v10" + "github.com/gojek/mlp/api/models/v2" "github.com/iancoleman/strcase" "github.com/knadh/koanf" "github.com/knadh/koanf/parsers/yaml" @@ -24,6 +25,7 @@ type Config struct { Streams Streams `validate:"dive,required"` Docs Documentations + Applications []models.Application `validate:"dive"` Authorization *AuthorizationConfig `validate:"required"` Database *DatabaseConfig `validate:"required"` Mlflow *MlflowConfig `validate:"required"` @@ -76,30 +78,24 @@ type UIConfig struct { StaticPath string `validated:"required"` IndexPath string `validated:"required"` - FeastCoreAPI string `json:"REACT_APP_FEAST_CORE_API"` - MerlinAPI string `json:"REACT_APP_MERLIN_API"` - TuringAPI string `json:"REACT_APP_TURING_API"` ClockworkUIHomepage string `json:"REACT_APP_CLOCKWORK_UI_HOMEPAGE"` - FeastUIHomepage string `json:"REACT_APP_FEAST_UI_HOMEPAGE"` KubeflowUIHomepage string `json:"REACT_APP_KUBEFLOW_UI_HOMEPAGE"` - MerlinUIHomepage string `json:"REACT_APP_MERLIN_UI_HOMEPAGE"` - TuringUIHomepage string `json:"REACT_APP_TURING_UI_HOMEPAGE"` } // Transform env variables to the format consumed by koanf. // The variable key is split by the double underscore ('__') sequence, // which separates nested config variables, and then each config key is -// converted to camel-case. +// converted to lower camel-case. // // Example: // // MY_VARIABLE => MyVariable -// VARIABLES__ANOTHER_VARIABLE => Variables.AnotherVariable +// VARIABLES__ANOTHER_VARIABLE => variables.anotherVariable func envVarKeyTransformer(s string) string { parts := strings.Split(strings.ToLower(s), "__") transformed := make([]string, len(parts)) for idx, key := range parts { - transformed[idx] = strcase.ToCamel(key) + transformed[idx] = strcase.ToLowerCamel(key) } return strings.Join(transformed, ".") @@ -154,7 +150,7 @@ func LoadAndValidate(paths ...string) (*Config, error) { } var defaultConfig = &Config{ - APIHost: "http://localhost:8080/v1", + APIHost: "http://localhost:8080", Environment: "dev", Port: 8080, diff --git a/api/config/config_test.go b/api/config/config_test.go index 86cf88f1..fe72ea29 100644 --- a/api/config/config_test.go +++ b/api/config/config_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/gojek/mlp/api/config" + "github.com/gojek/mlp/api/models/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,7 +49,7 @@ func TestLoad(t *testing.T) { configs: []string{"testdata/config-1.yaml"}, env: map[string]string{}, expected: &config.Config{ - APIHost: "http://localhost:8080/v1", + APIHost: "http://localhost:8080", Port: 8080, Environment: "dev", Authorization: &config.AuthorizationConfig{ @@ -81,7 +82,7 @@ func TestLoad(t *testing.T) { "DATABASE__PASSWORD": "secret", }, expected: &config.Config{ - APIHost: "http://localhost:8080/v1", + APIHost: "http://localhost:8080", Port: 8080, EncryptionKey: "test-key", Environment: "dev", @@ -119,12 +120,33 @@ func TestLoad(t *testing.T) { "SENTRY_DSN": "1234", }, expected: &config.Config{ - APIHost: "http://localhost:8080/v1", + APIHost: "http://localhost:8080", Port: 8080, EncryptionKey: "test-key", Environment: "dev", OauthClientID: "oauth-client-id", SentryDSN: "1234", + Applications: []models.Application{ + { + Name: "Turing", + Description: "ML Experimentation System", + Homepage: "/turing", + Configuration: &models.ApplicationConfig{ + API: "/api/turing/v1", + IconName: "graphApp", + Navigation: []models.NavigationMenuItem{ + { + Label: "Routers", + Destination: "/routers", + }, + { + Label: "Experiments", + Destination: "/experiments", + }, + }, + }, + }, + }, Authorization: &config.AuthorizationConfig{ Enabled: true, KetoServerURL: "http://localhost:4466", @@ -155,14 +177,8 @@ func TestLoad(t *testing.T) { StaticPath: "ui/build", IndexPath: "index.html", - FeastCoreAPI: "/feast/api", - MerlinAPI: "/api/merlin/v1", - TuringAPI: "/api/turing/v1", ClockworkUIHomepage: "http://clockwork.dev", - FeastUIHomepage: "/feast", KubeflowUIHomepage: "http://kubeflow.org", - MerlinUIHomepage: "/merlin", - TuringUIHomepage: "/turing", }, }, }, diff --git a/api/config/testdata/config-1.yaml b/api/config/testdata/config-1.yaml index 531209dc..b437ec7e 100644 --- a/api/config/testdata/config-1.yaml +++ b/api/config/testdata/config-1.yaml @@ -1,7 +1,7 @@ -Database: - User: mlp +database: + user: mlp -Streams: +streams: stream-1: - team-a - team-b diff --git a/api/config/testdata/config-2.yaml b/api/config/testdata/config-2.yaml index 3c83f5f3..1be5e667 100644 --- a/api/config/testdata/config-2.yaml +++ b/api/config/testdata/config-2.yaml @@ -1,17 +1,24 @@ -Docs: +docs: - label: "Merlin User Guide" href: "https://github.com/gojek/merlin/blob/main/docs/getting-started/README.md" -Authorization: - Enabled: true - KetoServerURL: http://localhost:4466 +applications: + - name: Turing + description: ML Experimentation System + homepage: /turing + configuration: + api: /api/turing/v1 + iconName: graphApp + navigation: + - label: Routers + destination: /routers + - label: Experiments + destination: /experiments -UI: - FeastCoreAPI: /feast/api - MerlinAPI: /api/merlin/v1 - TuringAPI: /api/turing/v1 - ClockworkUIHomepage: http://clockwork.dev - FeastUIHomepage: /feast - KubeflowUIHomepage: http://kubeflow.org - MerlinUIHomepage: /merlin - TuringUIHomepage: /turing +authorization: + enabled: true + ketoServerURL: http://localhost:4466 + +ui: + clockworkUIHomepage: http://clockwork.dev + kubeflowUIHomepage: http://kubeflow.org diff --git a/api/middleware/project_creation.go b/api/middleware/project_creation.go index a97f061e..33f7848f 100644 --- a/api/middleware/project_creation.go +++ b/api/middleware/project_creation.go @@ -5,9 +5,7 @@ import ( "strings" ) -type ProjectCreation struct{} - -func (a *ProjectCreation) ProjectCreationMiddleware(next http.Handler) http.Handler { +func ProjectCreationMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { userAgent := strings.ToLower(r.Header.Get("User-Agent")) diff --git a/api/models/v2/application.go b/api/models/v2/application.go new file mode 100644 index 00000000..215d09e8 --- /dev/null +++ b/api/models/v2/application.go @@ -0,0 +1,19 @@ +package models + +type Application struct { + Name string `json:"name" validate:"required"` + Description string `json:"description"` + Homepage string `json:"homepage"` + Configuration *ApplicationConfig `json:"config" validate:"dive"` +} + +type ApplicationConfig struct { + API string `json:"api"` + IconName string `json:"icon"` + Navigation []NavigationMenuItem `json:"navigation"` +} + +type NavigationMenuItem struct { + Label string `json:"label"` + Destination string `json:"destination"` +} diff --git a/api/static/swagger.yaml b/api/static/swagger.yaml index 2f9ce072..7331cd0e 100644 --- a/api/static/swagger.yaml +++ b/api/static/swagger.yaml @@ -4,7 +4,7 @@ info: description: "API Guide for accessing MLP API" version: "0.4.0" host: "localhost:8080" -basePath: "/v1" +basePath: "/" tags: - name: "project" description: "Project Management API. Project is used to namespace model, secret, and user access" @@ -13,7 +13,18 @@ tags: schemes: - "http" paths: - "/projects": + "/v2/applications": + get: + tags: ["application"] + summary: "List CaraML applications" + responses: + 200: + description: "OK" + schema: + type: "array" + items: + $ref: "#/definitions/Application" + "/v1/projects": get: tags: ["project"] summary: "List existing projects" @@ -49,7 +60,7 @@ paths: description: "Invalid request format" 409: description: "Project with the same name already exists" - "/projects/{project_id}": + "/v1/projects/{project_id}": get: tags: ["project"] summary: "Get project" @@ -89,7 +100,7 @@ paths: 400: description: "Invalid request format" - "/projects/{project_id}/secrets": + "/v1/projects/{project_id}/secrets": post: tags: ["secret"] summary: "Create secret" @@ -124,7 +135,7 @@ paths: items: $ref: "#/definitions/Secret" - "/projects/{project_id}/secrets/{secret_id}": + "/v1/projects/{project_id}/secrets/{secret_id}": patch: tags: ["secret"] summary: "Update secret" @@ -163,6 +174,35 @@ paths: description: "No content" definitions: + Application: + type: "object" + required: + - name + - homepage + properties: + name: + type: "string" + description: + type: "string" + homepage: + type: "string" + config: + type: "object" + properties: + api: + type: "string" + icon: + type: "string" + navigation: + type: "array" + items: + type: "object" + properties: + label: + type: "string" + destination: + type: "string" + Project: type: "object" required: diff --git a/config-dev.yaml b/config-dev.yaml index f00cecbf..497994dc 100644 --- a/config-dev.yaml +++ b/config-dev.yaml @@ -1,13 +1,61 @@ -EncryptionKey: "encryption-key" -OauthClientId: "[OAUTH_CLIENT_ID]" +encryptionKey: "encryption-key" +oauthClientId: "[OAUTH_CLIENT_ID]" -Database: - Host: "localhost" - Database: "mlp" - User: "mlp" - Password: "mlp" +applications: + - name: Merlin + description: Platform for deploying machine learning models + homepage: /merlin + configuration: + api: /api/merlin + iconName: machineLearningApp + navigation: + - label: Models + destination: /models + - label: Transformer Simulator + destination: /transformer-simulator + - name: Turing + description: Platform for setting up ML experiments + homepage: /turing + configuration: + api: /api/turing + iconName: graphApp + navigation: + - label: Routers + destination: /routers + - label: Ensemblers + destination: /ensemblers + - label: Ensembling Jobs + destination: /jobs + - label: Experiments + destination: /experiments + - name: Feast + description: Platform for managing and serving ML features + homepage: /feast + configuration: + api: /feast/api + iconName: appSearchApp + navigation: + - label: Entities + destination: /entities + - label: Feature Tables + destination: /featuretables + - label: Batch Ingestion Jobs + destination: /jobs/batch + - label: Stream Ingestion Jobs + destination: /jobs/stream + - name: Pipelines + description: Platform for managing ML pipelines + homepage: /pipeline + configuration: + iconName: pipelineApp -Docs: +database: + host: "localhost" + database: "mlp" + user: "mlp" + password: "mlp" + +docs: - label: "Merlin User Guide" href: "https://github.com/gojek/merlin/blob/main/docs/getting-started/README.md" - label: "Turing User Guide" @@ -15,10 +63,10 @@ Docs: - label: "Feast User Guide" href: "https://docs.feast.dev/user-guide/overview" -Mlflow: - TrackingURL: "http://mlflow.mlp.dev" +mlflow: + trackingURL: "http://mlflow.mlp.dev" -Streams: +streams: marketing: - growth - retention diff --git a/ui/packages/app/.env.development b/ui/packages/app/.env.development index c8ae2cb7..9404f1c6 100644 --- a/ui/packages/app/.env.development +++ b/ui/packages/app/.env.development @@ -1,6 +1,3 @@ -REACT_APP_API_URL=/v1 -REACT_APP_FEAST_CORE_API=/feast/api -REACT_APP_MERLIN_API=/api/merlin/v1 -REACT_APP_TURING_API=/api/turing/v1 +REACT_APP_API_URL=/ REACT_APP_OAUTH_CLIENT_ID=*** set client_id *** diff --git a/ui/packages/app/package.json b/ui/packages/app/package.json index d5e9fcc5..a4d11225 100644 --- a/ui/packages/app/package.json +++ b/ui/packages/app/package.json @@ -30,7 +30,7 @@ "start": "craco start", "test": "craco test" }, - "proxy": "http://localhost:8080/v1", + "proxy": "http://localhost:8080", "browserslist": { "production": [ ">0.2%", diff --git a/ui/packages/app/src/PrivateLayout.js b/ui/packages/app/src/PrivateLayout.js index bf097295..9fe8b93b 100644 --- a/ui/packages/app/src/PrivateLayout.js +++ b/ui/packages/app/src/PrivateLayout.js @@ -20,7 +20,7 @@ export const PrivateLayout = () => { {({ currentApp }) => (
- navigate(urlJoin(currentApp?.href, "projects", pId)) + navigate(urlJoin(currentApp?.homepage, "projects", pId)) } docLinks={config.DOC_LINKS} /> diff --git a/ui/packages/app/src/config.js b/ui/packages/app/src/config.js index a268523d..5ee30ce9 100644 --- a/ui/packages/app/src/config.js +++ b/ui/packages/app/src/config.js @@ -26,15 +26,8 @@ const config = { } ], - FEAST_CORE_API: getEnv("REACT_APP_FEAST_CORE_API"), // ${MLP_HOST}/feast/api - MERLIN_API: getEnv("REACT_APP_MERLIN_API"), // ${MLP_HOST}/api/merlin/v1 - TURING_API: getEnv("REACT_APP_TURING_API"), // ${MLP_HOST}/api/turing/v1 - CLOCKWORK_UI_HOMEPAGE: getEnv("REACT_APP_CLOCKWORK_UI_HOMEPAGE"), - FEAST_UI_HOMEPAGE: getEnv("REACT_APP_FEAST_UI_HOMEPAGE") || "/feast", - KUBEFLOW_UI_HOMEPAGE: getEnv("REACT_APP_KUBEFLOW_UI_HOMEPAGE"), - MERLIN_UI_HOMEPAGE: getEnv("REACT_APP_MERLIN_UI_HOMEPAGE") || "/merlin", - TURING_UI_HOMEPAGE: getEnv("REACT_APP_TURING_UI_HOMEPAGE") || "/turing" + KUBEFLOW_UI_HOMEPAGE: getEnv("REACT_APP_KUBEFLOW_UI_HOMEPAGE") }; export default config; diff --git a/ui/packages/app/src/hooks/useFeastCoreApi.js b/ui/packages/app/src/hooks/useFeastCoreApi.js index ae4697f9..d773caca 100644 --- a/ui/packages/app/src/hooks/useFeastCoreApi.js +++ b/ui/packages/app/src/hooks/useFeastCoreApi.js @@ -1,5 +1,5 @@ import { useContext } from "react"; -import { AuthContext, useApi } from "@gojek/mlp-ui"; +import { ApplicationsContext, AuthContext, useApi } from "@gojek/mlp-ui"; import config from "../config"; export const useFeastCoreApi = ( @@ -9,14 +9,12 @@ export const useFeastCoreApi = ( callImmediately = true ) => { const authCtx = useContext(AuthContext); - - /* Use undefined for authCtx so that the authorization header passed in the options - * will be used instead of being overwritten. Ref: https://github.com/gojek/mlp/blob/main/ui/packages/lib/src/utils/fetchJson.js#L39*/ + const { apps } = useContext(ApplicationsContext); return useApi( endpoint, { - baseApiUrl: config.FEAST_CORE_API, + baseApiUrl: apps.find(app => app.name === "Feast")?.config?.api, timeout: config.TIMEOUT, useMockData: config.USE_MOCK_DATA, ...options, @@ -27,6 +25,6 @@ export const useFeastCoreApi = ( }, authCtx, result, - callImmediately + apps.some(app => app.name === "Feast") && callImmediately ); }; diff --git a/ui/packages/app/src/hooks/useMerlinApi.js b/ui/packages/app/src/hooks/useMerlinApi.js index 0592dfc1..b36875c2 100644 --- a/ui/packages/app/src/hooks/useMerlinApi.js +++ b/ui/packages/app/src/hooks/useMerlinApi.js @@ -1,5 +1,5 @@ import { useContext } from "react"; -import { AuthContext, useApi } from "@gojek/mlp-ui"; +import { ApplicationsContext, AuthContext, useApi } from "@gojek/mlp-ui"; import config from "../config"; export const useMerlinApi = ( @@ -9,17 +9,18 @@ export const useMerlinApi = ( callImmediately = true ) => { const authCtx = useContext(AuthContext); + const { apps } = useContext(ApplicationsContext); return useApi( endpoint, { - baseApiUrl: config.MERLIN_API, + baseApiUrl: apps.find(app => app.name === "Merlin")?.config?.api, timeout: config.TIMEOUT, useMockData: config.USE_MOCK_DATA, ...options }, authCtx, result, - callImmediately + apps.some(app => app.name === "Merlin") && callImmediately ); }; diff --git a/ui/packages/app/src/hooks/useTuringApi.js b/ui/packages/app/src/hooks/useTuringApi.js index 19102861..7112794e 100644 --- a/ui/packages/app/src/hooks/useTuringApi.js +++ b/ui/packages/app/src/hooks/useTuringApi.js @@ -1,5 +1,5 @@ import { useContext } from "react"; -import { AuthContext, useApi } from "@gojek/mlp-ui"; +import { ApplicationsContext, AuthContext, useApi } from "@gojek/mlp-ui"; import config from "../config"; export const useTuringApi = ( @@ -9,17 +9,18 @@ export const useTuringApi = ( callImmediately = true ) => { const authCtx = useContext(AuthContext); + const { apps } = useContext(ApplicationsContext); return useApi( endpoint, { - baseApiUrl: config.TURING_API, + baseApiUrl: apps.find(app => app.name === "Turing")?.config?.api, timeout: config.TIMEOUT, useMockData: config.USE_MOCK_DATA, ...options }, authCtx, result, - callImmediately + apps.some(app => app.name === "Turing") && callImmediately ); }; diff --git a/ui/packages/app/src/pages/project/Project.js b/ui/packages/app/src/pages/project/Project.js index de7aa7e5..4849dde3 100644 --- a/ui/packages/app/src/pages/project/Project.js +++ b/ui/packages/app/src/pages/project/Project.js @@ -12,7 +12,6 @@ import { ProjectSummary } from "./components/ProjectSummary"; import { Resources } from "./components/Resources"; import { useMerlinApi } from "../../hooks/useMerlinApi"; import { useTuringApi } from "../../hooks/useTuringApi"; -import { useFeastCoreApi } from "../../hooks/useFeastCoreApi"; import { ComingSoonPanel } from "./components/ComingSoonPanel"; import imageCharts from "../../images/charts.svg"; @@ -22,42 +21,6 @@ const Project = () => { const { apps } = useContext(ApplicationsContext); const { currentProject } = useContext(ProjectsContext); - const [{ data: entities }] = useFeastCoreApi( - `/entities?project=${currentProject?.name}`, - { method: "GET" }, - undefined, - !!currentProject - ); - const [{ data: featureTables }] = useFeastCoreApi( - `/tables?project=${currentProject?.name}`, - { method: "GET" }, - undefined, - !!currentProject - ); - const [{ data: feastStreamIngestionJobs }] = useFeastCoreApi( - `/jobs/ingestion/stream`, - { - method: "POST", - body: JSON.stringify({ - include_terminated: true, - project: (currentProject?.name || "").replace(/-/g, "_") - }) - }, - undefined, - !!currentProject - ); - const [{ data: feastBatchIngestionJobs }] = useFeastCoreApi( - `/jobs/ingestion/batch`, - { - method: "POST", - body: JSON.stringify({ - include_terminated: true, - project: (currentProject?.name || "").replace(/-/g, "_") - }) - }, - undefined, - !!currentProject - ); const [{ data: models }] = useMerlinApi( `/projects/${currentProject?.id}/models`, { method: "GET" }, @@ -74,7 +37,7 @@ const Project = () => { return ( - {apps && !!currentProject ? ( + {!!currentProject ? ( <> @@ -84,8 +47,6 @@ const Project = () => { @@ -105,8 +66,6 @@ const Project = () => { diff --git a/ui/packages/app/src/pages/project/components/FeastJobsTable.js b/ui/packages/app/src/pages/project/components/FeastJobsTable.js index 29e1af50..729fdf14 100644 --- a/ui/packages/app/src/pages/project/components/FeastJobsTable.js +++ b/ui/packages/app/src/pages/project/components/FeastJobsTable.js @@ -1,11 +1,33 @@ import React, { useEffect, useState } from "react"; import { EuiInMemoryTable } from "@elastic/eui"; +import { useFeastCoreApi } from "../../../hooks/useFeastCoreApi"; + +export const FeastJobsTable = ({ project, homepage }) => { + const [{ data: feastStreamIngestionJobs }] = useFeastCoreApi( + `/jobs/ingestion/stream`, + { + method: "POST", + body: JSON.stringify({ + include_terminated: true, + project: (project?.name || "").replace(/-/g, "_") + }) + }, + [], + !!project + ); + const [{ data: feastBatchIngestionJobs }] = useFeastCoreApi( + `/jobs/ingestion/batch`, + { + method: "POST", + body: JSON.stringify({ + include_terminated: true, + project: (project?.name || "").replace(/-/g, "_") + }) + }, + [], + !!project + ); -export const FeastJobsTable = ({ - project, - feastStreamIngestionJobs, - feastBatchIngestionJobs -}) => { const [items, setItems] = useState([]); useEffect(() => { @@ -30,7 +52,6 @@ export const FeastJobsTable = ({ } if (feastBatchIngestionJobs) { - console.log("feastBatchIngestionJobs", feastBatchIngestionJobs); feastBatchIngestionJobs .filter(job => job.status === "IN PROGRESS") .sort((a, b) => (a.startTime > b.startTime ? 1 : -1)) @@ -77,7 +98,7 @@ export const FeastJobsTable = ({ const cellProps = item => ({ style: { cursor: "pointer" }, onClick: () => - (window.location.href = `/feast/projects/${project.id}/jobs/${item.type}`) + (window.location.href = `${homepage}/projects/${project.id}/jobs/${item.type}`) }); return ( diff --git a/ui/packages/app/src/pages/project/components/FeastResources.js b/ui/packages/app/src/pages/project/components/FeastResources.js index bbc6b9e7..108f2d18 100644 --- a/ui/packages/app/src/pages/project/components/FeastResources.js +++ b/ui/packages/app/src/pages/project/components/FeastResources.js @@ -1,45 +1,49 @@ import React, { useEffect, useState } from "react"; import { EuiListGroup, EuiText } from "@elastic/eui"; +import { useFeastCoreApi } from "../../../hooks/useFeastCoreApi"; import "./ListGroup.scss"; -export const FeastResources = ({ project, entities, featureTables }) => { +export const FeastResources = ({ project, homepage }) => { + const [{ data: entities }] = useFeastCoreApi( + `/entities?project=${project?.name}`, + { method: "GET" }, + { entities: [] }, + !!project + ); + const [{ data: featureTables }] = useFeastCoreApi( + `/tables?project=${project?.name}`, + { method: "GET" }, + { tables: [] }, + !!project + ); + const [items, setItems] = useState([]); useEffect(() => { - if ( - project && - entities && - entities.entities && - featureTables && - featureTables.tables - ) { - setItems([ - { - className: "listGroupItem", - label: ( - {entities.entities.length} entities - ), - onClick: () => { - window.location.href = `/feast/projects/${project.id}/entities`; - }, - size: "s" + setItems([ + { + className: "listGroupItem", + label: {entities.entities.length} entities, + onClick: () => { + window.location.href = `${homepage}/projects/${project.id}/entities`; + }, + size: "s" + }, + { + className: "listGroupItem", + label: ( + + {featureTables.tables.length} feature tables + + ), + onClick: () => { + window.location.href = `${homepage}/projects/${project.id}/featuretables`; }, - { - className: "listGroupItem", - label: ( - - {featureTables.tables.length} feature tables - - ), - onClick: () => { - window.location.href = `/feast/projects/${project.id}/featuretables`; - }, - size: "s" - } - ]); - } - }, [project, entities, featureTables]); + size: "s" + } + ]); + }, [project, entities, featureTables, homepage]); return items.length > 0 ? ( { return ( @@ -22,42 +23,72 @@ const Title = ({ title, href }) => { ); }; -export const Instances = ({ - project, - feastStreamIngestionJobs, - feastBatchIngestionJobs, - models, - routers -}) => { +export const Instances = ({ project, models, routers }) => { + const { apps } = useContext(ApplicationsContext); + const items = [ - { - title: , - description: ( - <FeastJobsTable - project={project} - feastStreamIngestionJobs={feastStreamIngestionJobs} - feastBatchIngestionJobs={feastBatchIngestionJobs} - /> - ) - }, - { - title: ( - <Title - title="Merlin Deployments" - href={`/merlin/projects/${project.id}/models`} - /> - ), - description: <MerlinDeploymentsTable project={project} models={models} /> - }, - { - title: ( - <Title - title="Turing Routers" - href={`/turing/projects/${project.id}/routers`} - /> - ), - description: <TuringRoutersTable project={project} routers={routers} /> - } + ...(apps.some(a => a.name === "Feast") + ? [ + { + title: ( + <Title + title="Features Ingestion" + href={`${ + apps.find(a => a.name === "Feast").homepage + }/projects/${project.id}/jobs/stream`} + /> + ), + description: ( + <FeastJobsTable + project={project} + homepage={apps.find(a => a.name === "Feast").homepage} + /> + ) + } + ] + : []), + ...(apps.some(a => a.name === "Merlin") + ? [ + { + title: ( + <Title + title="Merlin Deployments" + href={`${ + apps.find(a => a.name === "Merlin").homepage + }/projects/${project.id}/models`} + /> + ), + description: ( + <MerlinDeploymentsTable + project={project} + models={models} + homepage={apps.find(a => a.name === "Merlin").homepage} + /> + ) + } + ] + : []), + ...(apps.some(a => a.name === "Turing") + ? [ + { + title: ( + <Title + title="Turing Routers" + href={`${ + apps.find(a => a.name === "Turing").homepage + }/projects/${project.id}/routers`} + /> + ), + description: ( + <TuringRoutersTable + project={project} + routers={routers} + homepage={apps.find(a => a.name === "Turing").homepage} + /> + ) + } + ] + : []) ]; return ( diff --git a/ui/packages/app/src/pages/project/components/MerlinDeploymentsTable.js b/ui/packages/app/src/pages/project/components/MerlinDeploymentsTable.js index 8319a630..e3e53caa 100644 --- a/ui/packages/app/src/pages/project/components/MerlinDeploymentsTable.js +++ b/ui/packages/app/src/pages/project/components/MerlinDeploymentsTable.js @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; import { EuiInMemoryTable } from "@elastic/eui"; -export const MerlinDeploymentsTable = ({ project, models }) => { +export const MerlinDeploymentsTable = ({ project, models, homepage }) => { const [items, setItems] = useState([]); useEffect(() => { @@ -59,7 +59,7 @@ export const MerlinDeploymentsTable = ({ project, models }) => { const cellProps = item => ({ style: { cursor: "pointer" }, onClick: () => - (window.location.href = `/merlin/projects/${project.id}/models/${item.id}/versions/${item.version_id}/endpoints/${item.version_endpoint_id}/details`) + (window.location.href = `${homepage}/projects/${project.id}/models/${item.id}/versions/${item.version_id}/endpoints/${item.version_endpoint_id}/details`) }); return ( diff --git a/ui/packages/app/src/pages/project/components/MerlinModels.js b/ui/packages/app/src/pages/project/components/MerlinModels.js index 84d694c8..3adf43e7 100644 --- a/ui/packages/app/src/pages/project/components/MerlinModels.js +++ b/ui/packages/app/src/pages/project/components/MerlinModels.js @@ -4,7 +4,7 @@ import { MODEL_TYPE_NAME_MAP } from "../../../services/merlin/Model"; import "./ListGroup.scss"; -export const MerlinModels = ({ project, models }) => { +export const MerlinModels = ({ project, models, homepage }) => { const [modelItems, setModelItems] = useState([]); useEffect(() => { @@ -31,14 +31,14 @@ export const MerlinModels = ({ project, models }) => { </EuiText> ), onClick: () => { - window.location.href = `/merlin/projects/${project.id}/models?type=${modelType}`; + window.location.href = `${homepage}/projects/${project.id}/models?type=${modelType}`; }, size: "s" }); }); setModelItems(items); } - }, [project, models]); + }, [project, models, homepage]); return modelItems.length > 0 ? ( <EuiListGroup diff --git a/ui/packages/app/src/pages/project/components/Resources.js b/ui/packages/app/src/pages/project/components/Resources.js index b559c694..397934eb 100644 --- a/ui/packages/app/src/pages/project/components/Resources.js +++ b/ui/packages/app/src/pages/project/components/Resources.js @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React from "react"; import { EuiContextMenuItem, EuiContextMenuPanel, @@ -12,23 +12,10 @@ import { Panel } from "./Panel"; import { FeastResources } from "./FeastResources"; import { MerlinModels } from "./MerlinModels"; import { TuringRouters } from "./TuringRouters"; +import { useToggle } from "@gojek/mlp-ui"; -export const Resources = ({ - project, - entities, - featureTables, - models, - routers -}) => { - const [isPopoverOpen, setPopover] = useState(false); - - const onButtonClick = () => { - setPopover(!isPopoverOpen); - }; - - const closePopover = () => { - setPopover(false); - }; +export const Resources = ({ apps, project, models, routers }) => { + const [isPopoverOpen, togglePopover] = useToggle(false); const newResourceMenus = [ <EuiContextMenuItem @@ -37,54 +24,97 @@ export const Resources = ({ size="s"> Create notebook </EuiContextMenuItem>, - <EuiContextMenuItem - href={`${config.FEAST_UI_HOMEPAGE}/projects/${project.id}/entities/create`} - key="feature-table" - size="s"> - Create Entities - </EuiContextMenuItem>, - <EuiContextMenuItem - href={`${config.FEAST_UI_HOMEPAGE}/projects/${project.id}/featuretables/create`} - key="feature-table" - size="s"> - Create FeatureTable - </EuiContextMenuItem>, - <EuiContextMenuItem - href={`${config.MERLIN_UI_HOMEPAGE}/projects/${project.id}`} - key="model" - size="s"> - Deploy model - </EuiContextMenuItem>, - <EuiContextMenuItem - href={`${config.TURING_UI_HOMEPAGE}/projects/${project.id}/routers/create`} - key="experiment" - size="s"> - Set up experiment - </EuiContextMenuItem>, + ...(apps.some(a => a.name === "Feast") + ? [ + <EuiContextMenuItem + href={`${apps.find(a => a.name === "Feast")?.homepage}/projects/${ + project.id + }/entities/create`} + key="feature-table" + size="s"> + Create Entities + </EuiContextMenuItem>, + <EuiContextMenuItem + href={`${apps.find(a => a.name === "Feast")?.homepage}/projects/${ + project.id + }/featuretables/create`} + key="feature-table" + size="s"> + Create FeatureTable + </EuiContextMenuItem> + ] + : []), + ...(apps.some(a => a.name === "Merlin") + ? [ + <EuiContextMenuItem + href={`${apps.find(a => a.name === "Merlin")?.homepage}/projects/${ + project.id + }`} + key="model" + size="s"> + Deploy model + </EuiContextMenuItem> + ] + : []), + ...(apps.some(a => a.name === "Turing") + ? [ + <EuiContextMenuItem + href={`${apps.find(a => a.name === "Turing")?.homepage}/projects/${ + project.id + }/routers/create`} + key="experiment" + size="s"> + Set up experiment + </EuiContextMenuItem> + ] + : []), <EuiContextMenuItem href={config.CLOCKWORK_UI_HOMEPAGE} key="job" size="s"> Schedule a job </EuiContextMenuItem> ]; const items = [ - { - title: "Features", - description: ( - <FeastResources - project={project} - entities={entities} - featureTables={featureTables} - /> - ) - }, - { - title: "Models", - description: <MerlinModels project={project} models={models} /> - }, - { - title: "Experiments", - description: <TuringRouters project={project} routers={routers} /> - }, + ...(apps.some(a => a.name === "Feast") + ? [ + { + title: "Features", + description: ( + <FeastResources + project={project} + homepage={apps.find(a => a.name === "Feast")?.homepage} + /> + ) + } + ] + : []), + ...(apps.some(a => a.name === "Merlin") + ? [ + { + title: "Models", + description: ( + <MerlinModels + project={project} + models={models} + homepage={apps.find(a => a.name === "Merlin")?.homepage} + /> + ) + } + ] + : []), + ...(apps.some(a => a.name === "Turing") + ? [ + { + title: "Experiments", + description: ( + <TuringRouters + project={project} + routers={routers} + homepage={apps.find(a => a.name === "Turing")?.homepage} + /> + ) + } + ] + : []), { title: "Notebooks", description: ( @@ -104,14 +134,14 @@ export const Resources = ({ const actions = ( <EuiPopover button={ - <EuiLink onClick={onButtonClick}> + <EuiLink onClick={togglePopover}> <EuiText size="s"> <EuiIcon type="arrowRight" /> Create a new resource </EuiText> </EuiLink> } isOpen={isPopoverOpen} - closePopover={closePopover} + closePopover={togglePopover} anchorPosition="rightUp" offset={10} panelPaddingSize="s"> diff --git a/ui/packages/app/src/pages/project/components/TuringRouters.js b/ui/packages/app/src/pages/project/components/TuringRouters.js index f84bba8d..dfe1a606 100644 --- a/ui/packages/app/src/pages/project/components/TuringRouters.js +++ b/ui/packages/app/src/pages/project/components/TuringRouters.js @@ -4,7 +4,7 @@ import { EXPERIMENT_TYPE_NAME_MAP } from "../../../services/turing/Turing"; import "./ListGroup.scss"; -export const TuringRouters = ({ project, routers }) => { +export const TuringRouters = ({ project, routers, homepage }) => { const [experiments, setExperiments] = useState([]); useEffect(() => { @@ -36,14 +36,14 @@ export const TuringRouters = ({ project, routers }) => { </EuiText> ), onClick: () => { - window.location.href = `/turing/projects/${project.id}/routers?experiment_type=${expType}`; + window.location.href = `${homepage}/projects/${project.id}/routers?experiment_type=${expType}`; }, size: "s" }); }); setExperiments(exps); } - }, [project, routers]); + }, [project, routers, homepage]); return experiments.length > 0 ? ( <EuiListGroup diff --git a/ui/packages/app/src/pages/project/components/TuringRoutersTable.js b/ui/packages/app/src/pages/project/components/TuringRoutersTable.js index c78cf522..2894b7d2 100644 --- a/ui/packages/app/src/pages/project/components/TuringRoutersTable.js +++ b/ui/packages/app/src/pages/project/components/TuringRoutersTable.js @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; import { EuiInMemoryTable } from "@elastic/eui"; -export const TuringRoutersTable = ({ project, routers }) => { +export const TuringRoutersTable = ({ project, routers, homepage }) => { const [items, setItems] = useState([]); useEffect(() => { @@ -48,7 +48,7 @@ export const TuringRoutersTable = ({ project, routers }) => { const cellProps = item => ({ style: { cursor: "pointer" }, onClick: () => - (window.location.href = `/turing/projects/${project.id}/routers/${item.id}/details`) + (window.location.href = `${homepage}/projects/${project.id}/routers/${item.id}/details`) }); return ( diff --git a/ui/packages/app/src/project_setting/SecretSetting.js b/ui/packages/app/src/project_setting/SecretSetting.js index 95a6a840..7b48d171 100644 --- a/ui/packages/app/src/project_setting/SecretSetting.js +++ b/ui/packages/app/src/project_setting/SecretSetting.js @@ -7,7 +7,7 @@ import { useParams } from "react-router-dom"; const SecretSetting = () => { const { projectId } = useParams(); const [{ data, isLoaded, error }, fetchSecrets] = useMlpApi( - `/projects/${projectId}/secrets` + `/v1/projects/${projectId}/secrets` ); return ( diff --git a/ui/packages/app/src/project_setting/form/ProjectForm.js b/ui/packages/app/src/project_setting/form/ProjectForm.js index 704a3d05..4a336d33 100644 --- a/ui/packages/app/src/project_setting/form/ProjectForm.js +++ b/ui/packages/app/src/project_setting/form/ProjectForm.js @@ -149,7 +149,7 @@ const ProjectForm = () => { submitForm({ body: JSON.stringify(project) }); }; const [submissionResponse, submitForm] = useMlpApi( - "/projects", + "/v1/projects", { method: "POST", headers: { "Content-Type": "application/json" } diff --git a/ui/packages/app/src/project_setting/secret/DeleteSecretModal.js b/ui/packages/app/src/project_setting/secret/DeleteSecretModal.js index d4e752f9..f5268469 100644 --- a/ui/packages/app/src/project_setting/secret/DeleteSecretModal.js +++ b/ui/packages/app/src/project_setting/secret/DeleteSecretModal.js @@ -4,7 +4,7 @@ import { EuiConfirmModal, EuiOverlayMask } from "@elastic/eui"; const DeleteSecretModal = ({ projectId, secret, closeModal, fetchUpdates }) => { const [deleteResponse, deleteSecret] = useMlpApi( - `/projects/${projectId}/secrets/${secret.id}`, + `/v1/projects/${projectId}/secrets/${secret.id}`, { method: "DELETE" }, diff --git a/ui/packages/app/src/project_setting/secret/SubmitSecretForm.js b/ui/packages/app/src/project_setting/secret/SubmitSecretForm.js index 8c5c75bd..fdbcbf33 100644 --- a/ui/packages/app/src/project_setting/secret/SubmitSecretForm.js +++ b/ui/packages/app/src/project_setting/secret/SubmitSecretForm.js @@ -29,8 +29,8 @@ const SubmitSecretForm = ({ projectId, fetchUpdates, secret, toggleAdd }) => { const [submissionResponse, submitForm] = useMlpApi( secret - ? `/projects/${projectId}/secrets/${secret.id}` - : `/projects/${projectId}/secrets`, + ? `/v1/projects/${projectId}/secrets/${secret.id}` + : `/v1/projects/${projectId}/secrets`, { method: secret ? "PATCH" : "POST", headers: { "Content-Type": "application/json" } diff --git a/ui/packages/app/src/project_setting/user_role/DeleteUserRoleModal.js b/ui/packages/app/src/project_setting/user_role/DeleteUserRoleModal.js index 27d69806..bec7d21c 100644 --- a/ui/packages/app/src/project_setting/user_role/DeleteUserRoleModal.js +++ b/ui/packages/app/src/project_setting/user_role/DeleteUserRoleModal.js @@ -9,7 +9,7 @@ const DeleteUserRoleModal = ({ fetchUpdates }) => { const [deleteResponse, deleteUserRole] = useMlpApi( - `/projects/${project.id}`, + `/v1/projects/${project.id}`, { method: "PUT" }, diff --git a/ui/packages/app/src/project_setting/user_role/SubmitUserRoleForm.js b/ui/packages/app/src/project_setting/user_role/SubmitUserRoleForm.js index 5487b37c..bd165611 100644 --- a/ui/packages/app/src/project_setting/user_role/SubmitUserRoleForm.js +++ b/ui/packages/app/src/project_setting/user_role/SubmitUserRoleForm.js @@ -26,7 +26,7 @@ const SubmitUserRoleForm = ({ userRole, project, fetchUpdates, toggleAdd }) => { }); const [submissionResponse, submitForm] = useMlpApi( - `/projects/${project.id}`, + `/v1/projects/${project.id}`, { method: "PUT", headers: { "Content-Type": "application/json" } diff --git a/ui/packages/app/src/services/merlin/Merlin.js b/ui/packages/app/src/services/merlin/Merlin.js deleted file mode 100644 index 283ce966..00000000 --- a/ui/packages/app/src/services/merlin/Merlin.js +++ /dev/null @@ -1,16 +0,0 @@ -import { useContext, useEffect, useState } from "react"; -import { ApplicationsContext } from "@gojek/mlp-ui"; - -export const MerlinApp = () => { - const { apps } = useContext(ApplicationsContext); - const [merlinApp, setMerlinApp] = useState({}); - - useEffect(() => { - if (apps) { - const merlin = apps.find(app => app.name === "Merlin"); - setMerlinApp(merlin); - } - }, [apps]); - - return merlinApp; -}; diff --git a/ui/packages/lib/README.md b/ui/packages/lib/README.md index caff3551..8b7d54bf 100644 --- a/ui/packages/lib/README.md +++ b/ui/packages/lib/README.md @@ -62,7 +62,7 @@ const DummyButton = () => { ```js const [response, fetch] = useMlpApi( - `/projects/${projectId}/environments`, + `/v1/projects/${projectId}/environments`, {}, // request options [], // initial value of the response.data true // whether or not to send the request to the API immediately diff --git a/ui/packages/lib/src/components/nav_drawer/NavDrawer.js b/ui/packages/lib/src/components/nav_drawer/NavDrawer.js index 51223622..b95459fe 100644 --- a/ui/packages/lib/src/components/nav_drawer/NavDrawer.js +++ b/ui/packages/lib/src/components/nav_drawer/NavDrawer.js @@ -30,17 +30,16 @@ export const NavDrawer = ({ docLinks }) => { return apps.map(a => { const isAppActive = a === currentApp; - const children = !!currentProject - ? a?.config?.sections.map(s => ({ - id: slugify(`${a.name}.${s.name}`), - label: s.name, + ? a?.config?.navigation?.map(s => ({ + id: slugify(`${a.name}.${s.label}`), + label: s.label, callback: () => { const dest = urlJoin( - a.href, + a.homepage, "projects", currentProject.id, - s.href + s.destination ); isAppActive ? navigate(dest) : (window.location.href = dest); @@ -52,7 +51,7 @@ export const NavDrawer = ({ docLinks }) => { return { id: slugify(a.name), label: a.name, - icon: <EuiIcon type={a.icon} />, + icon: <EuiIcon type={a.config.icon} />, isExpanded: isAppActive || isRootApplication, className: isAppActive ? "euiTreeView__node---small---active" @@ -61,8 +60,8 @@ export const NavDrawer = ({ docLinks }) => { callback: () => !children || !currentProject ? (window.location.href = !!currentProject - ? urlJoin(a.href, "projects", currentProject.id) - : a.href) + ? urlJoin(a.homepage, "projects", currentProject.id) + : a.homepage) : {}, children: children }; diff --git a/ui/packages/lib/src/hooks/useApi.js b/ui/packages/lib/src/hooks/useApi.js index 32687cac..77b13caf 100644 --- a/ui/packages/lib/src/hooks/useApi.js +++ b/ui/packages/lib/src/hooks/useApi.js @@ -67,7 +67,8 @@ export const useApi = ( const [args, dispatchArgsUpdate] = useReducer(argumentsReducer, { result, options, - authCtx + authCtx, + callImmediately }); useEffect(() => { @@ -82,6 +83,10 @@ export const useApi = ( dispatchArgsUpdate({ name: "result", value: result }); }, [result]); + useEffect(() => { + dispatchArgsUpdate({ name: "callImmediately", value: callImmediately }); + }, [callImmediately]); + const [state, dispatch] = useReducer( dataFetchReducer, args.result, @@ -139,11 +144,11 @@ export const useApi = ( useEffect(() => { dispatch({ type: "FETCH_RESET", payload: args.result }); - if (callImmediately) { + if (args.callImmediately) { const call = fetchData(); return call.cancel; } - }, [args.result, callImmediately, fetchData]); + }, [args.result, args.callImmediately, fetchData]); return [state, fetchData]; }; diff --git a/ui/packages/lib/src/providers/application/context.js b/ui/packages/lib/src/providers/application/context.js index 0e96a7b6..d254fde7 100644 --- a/ui/packages/lib/src/providers/application/context.js +++ b/ui/packages/lib/src/providers/application/context.js @@ -1,5 +1,5 @@ import React, { useMemo } from "react"; -import { useMlpApi } from "../../hooks/useMlpApi"; +import { useMlpApi } from "../../hooks"; import { useLocation } from "react-router-dom"; const ApplicationsContext = React.createContext({ @@ -9,10 +9,10 @@ const ApplicationsContext = React.createContext({ export const ApplicationsContextProvider = ({ children }) => { const location = useLocation(); - const [{ data: apps }] = useMlpApi("/applications", {}, []); + const [{ data: apps }] = useMlpApi("/v2/applications", {}, []); const currentApp = useMemo( - () => apps.find(a => location.pathname.startsWith(a.href)), + () => apps.find(a => location.pathname.startsWith(a.homepage)), [apps, location.pathname] ); diff --git a/ui/packages/lib/src/providers/project/context.js b/ui/packages/lib/src/providers/project/context.js index df83d5f2..a2df5649 100644 --- a/ui/packages/lib/src/providers/project/context.js +++ b/ui/packages/lib/src/providers/project/context.js @@ -31,12 +31,12 @@ const Context = React.createContext({ }); export const ProjectsContextProvider = ({ children }) => { - const [{ data: projects }, refresh] = useMlpApi(`/projects`, {}, []); + const [{ data: projects }, refresh] = useMlpApi(`/v1/projects`, {}, []); const { currentApp = {} } = useContext(ApplicationsContext); const projectIdMatch = useMatch({ - path: urlJoin(currentApp.href, "/projects/:projectId"), + path: urlJoin(currentApp.homepage, "/projects/:projectId"), caseSensitive: true, end: false });