Support reading configuration from config files

.env.development

@@ -1,14 +1,4 @@
 DATABASE_HOST=localhost
 DATABASE_USER=mlp
 DATABASE_PASSWORD=mlp
-DATABASE_NAME=mlp
-
-ENCRYPTION_KEY="password"
-
-MLFLOW_TRACKING_URL=http://mlflow.mlp.dev
-
-OAUTH_CLIENT_ID=[OAUTH_CLIENT_ID]
-
-TEAM_LIST=business,marketing,operation
-STREAM_LIST=business-strategy,promotion-marketing,warehouse
-DOC_LIST=[{"href":"https://github.com/gojek/merlin/blob/main/docs/getting-started/README.md","label":"Merlin User Guide"},{"href":"https://github.com/gojek/turing","label":"Turing User Guide"},{"href":"https://docs.feast.dev/user-guide/overview","label":"Feast User Guide"}]
+DATABASE_NAME=mlp

.github/e2e/values.yaml

@@ -1,4 +1,7 @@
 # This values only used for E2E tests.
+fullnameOverride: mlp
+nameOverride: mlp
+
 deployment:
   environment: dev
   resources:

.github/workflows/ci-pipeline.yml

@@ -27,6 +27,9 @@ env:
   CONTAINER_REGISTRY: ghcr.io
   LOCAL_REGISTRY: registry.localhost:5000
 
+  CARAML_CHARTS_REPOSITORY: romanwozniak/helm-charts
+  CARAML_CHARTS_REF: feat-mlp-support-configmap
+
 jobs:
   build-ui:
     runs-on: ubuntu-latest
@@ -68,7 +71,7 @@ jobs:
     outputs:
       api-version: ${{ steps.build-image.outputs.api-version }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
@@ -150,6 +153,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ env.CARAML_CHARTS_REPOSITORY }}
+          ref: ${{ env.CARAML_CHARTS_REF }}
+          path: 'helm-charts'
+
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ env.GO_VERSION }}
@@ -186,23 +195,39 @@ jobs:
           docker push ${{ env.DOCKER_REPOSITORY }}/mlp-api:${{ needs.build-api.outputs.api-version }}
 
       - name: Deploy MLP to the local cluster
+        id: deploy
         run: |
-          helm repo add caraml https://caraml-dev.github.io/helm-charts/
+          helm install mlp ./helm-charts/charts/mlp \
+            --dependency-update \
+            --wait --timeout=3m \
+            --values=.github/e2e/values.yaml \
+            --set deployment.image.registry=${{ env.LOCAL_REGISTRY }} \
+            --set deployment.image.repository=${{ github.repository }}/mlp-api \
+            --set deployment.image.tag=${{ needs.build-api.outputs.api-version }}
+
+      - if: steps.deploy.outcome == 'failure' && always()
+        name: "Debug Deployment Failure"
+        run: |
+          echo "::group::describe deployment/mlp"
+          kubectl describe deployment/mlp
+          echo "::endgroup::"
 
-          kubectl create ns mlp
+          echo "::group::configmap/mlp-config"
+          kubectl get cm/mlp-config -o jsonpath='{.data.mlp-config\.yaml}'
+          echo "::endgroup::"
 
-          helm install mlp caraml/mlp --namespace=mlp \
-            --wait --timeout=5m \
-            --values=.github/e2e/values.yaml \
-            --set mlp.image.registry=${{ env.LOCAL_REGISTRY }} \
-            --set mlp.image.repository=${{ github.repository }}/mlp-api \
-            --set mlp.image.tag=${{ needs.build-api.outputs.api-version }}
+          echo "::group::logs deployment/mlp"
+          kubectl logs deployment/mlp
+          echo "::endgroup::"
 
-          echo "::group::Check cluster resources"
-          kubectl get all --namespace=mlp
-          kubectl get ingress --namespace=mlp
+          echo "::group::kubernetes events"
+          kubectl get events
           echo "::endgroup::"
 
+          echo "::group::kubernetes pod describe"
+          kubectl describe pod
+          echo "::endgroup::"      
+
       - name: Setup MLP project
         shell: bash
         run: |

Dockerfile

@@ -22,9 +22,9 @@ RUN go build -o bin/mlp-api ./api/cmd/main.go
 # ============================================================
 # Build stage 3: Run the app
 # ============================================================
-FROM alpine:3.12
+FROM alpine:3.16
 COPY --from=node-builder /src/ui/build ./ui/build
 COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp
 COPY --from=go-builder /src/api/db-migrations ./db-migrations
 
-CMD ["mlp"]
+ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]

Makefile

@@ -6,6 +6,8 @@ UI_BUILD_PATH := ${UI_PATH}/build
 API_PATH := api
 API_ALL_PACKAGES := $(shell cd ${API_PATH} && go list ./... | grep -v github.com/gojek/mlp/client | grep -v mocks)
 BIN_NAME := $(if ${APP_NAME},${APP_NAME},mlp)
+DOCKER_COMPOSE_BIN := "docker-compose"
+DOCKER_COMPOSE_ARG := ""
 
 all: setup init-dep lint test clean build run
 
@@ -117,7 +119,7 @@ build-image: version
 .PHONY: run
 run: build-api local-db
 	@echo "> Running application ..."
-	@./bin/${BIN_NAME}
+	@./bin/${BIN_NAME} --config config-dev.yaml
 
 .PHONY: start-ui
 start-ui:
@@ -148,22 +150,22 @@ generate-client:
 .PHONY: local-db
 local-db:
 	@echo "> Starting up DB ..."
-	@docker-compose up -d postgres
+	@$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) up -d postgres
 
 .PHONY: start-keto
 start-keto:
 	@echo "> Starting up keto server ..."
-	@docker-compose up -d keto
+	@$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) up -d keto
 
 .PHONY: stop-docker
 stop-docker:
 	@echo "> Stopping Docker compose ..."
-	@docker-compose down
+	@$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) down
 
 .PHONY: swagger-ui
 swagger-ui:
 	@echo "> Starting up Swagger UI ..."
-	@docker-compose up -d swagger-ui
+	@$(DOCKER_COMPOSE_BIN) $(DOCKER_COMPOSE_ARG) up -d swagger-ui
 
 .PHONY: version
 version:

api.Dockerfile

@@ -18,4 +18,4 @@ FROM alpine:3.16
 COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp
 COPY db-migrations ./db-migrations
 
-CMD ["mlp"]
+ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]

api/cmd/main.go

@@ -8,43 +8,46 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/gojek/mlp/api/pkg/authz/enforcer"
-	"github.com/gorilla/mux"
-	"github.com/heptiolabs/healthcheck"
-	"github.com/rs/cors"
-
 	"github.com/gojek/mlp/api/api"
 	"github.com/gojek/mlp/api/config"
 	"github.com/gojek/mlp/api/database"
 	"github.com/gojek/mlp/api/log"
+	"github.com/gojek/mlp/api/pkg/authz/enforcer"
 	"github.com/gojek/mlp/api/service"
 	"github.com/gojek/mlp/api/storage"
+	"github.com/gorilla/mux"
+	"github.com/heptiolabs/healthcheck"
+	"github.com/rs/cors"
+	flag "github.com/spf13/pflag"
 )
 
 func main() {
-	cfg, err := config.InitConfigEnv()
+	configFiles := flag.StringSliceP("config", "c", []string{}, "Path to a configuration files")
+	flag.Parse()
+
+	cfg, err := config.LoadAndValidate(*configFiles...)
 	if err != nil {
 		log.Panicf("Failed initializing config: %v", err)
 	}
 
 	// init db
-	db, err := database.InitDB(&cfg.DbConfig)
+	db, err := database.InitDB(cfg.Database)
 	if err != nil {
 		panic(err)
 	}
 	defer db.Close()
 
 	applicationService, _ := service.NewApplicationService(db)
 	authEnforcer, _ := enforcer.NewEnforcerBuilder().
-		URL(cfg.AuthorizationConfig.AuthorizationServerURL).
+		URL(cfg.Authorization.KetoServerURL).
 		Product("mlp").
 		Build()
 
 	projectsService, err := service.NewProjectsService(
-		cfg.MlflowConfig.TrackingURL,
+		cfg.Mlflow.TrackingURL,
 		storage.NewProjectStorage(db),
 		authEnforcer,
-		cfg.AuthorizationConfig.AuthorizationEnabled)
+		cfg.Authorization.Enabled)
 
 	if err != nil {
 		log.Panicf("unable to initialize project service: %v", err)
@@ -57,7 +60,7 @@ func main() {
 		ProjectsService:    projectsService,
 		SecretService:      secretService,
 
-		AuthorizationEnabled: cfg.AuthorizationConfig.AuthorizationEnabled,
+		AuthorizationEnabled: cfg.Authorization.Enabled,
 		Enforcer:             authEnforcer,
 	}
 
@@ -70,11 +73,9 @@ func main() {
 		OauthClientID: cfg.OauthClientID,
 		Environment:   cfg.Environment,
 		SentryDSN:     cfg.SentryDSN,
-		Teams:         cfg.Teams,
 		Streams:       cfg.Streams,
 		Docs:          cfg.Docs,
-
-		UIConfig: cfg.UI,
+		UIConfig:      cfg.UI,
 	}
 
 	router.Methods("GET").Path("/env.js").HandlerFunc(uiEnv.handler)
@@ -96,15 +97,14 @@ func mount(r *mux.Router, path string, handler http.Handler) {
 }
 
 type uiEnvHandler struct {
+	*config.UIConfig
+
 	APIURL        string                `json:"REACT_APP_API_URL,omitempty"`
 	OauthClientID string                `json:"REACT_APP_OAUTH_CLIENT_ID,omitempty"`
 	Environment   string                `json:"REACT_APP_ENVIRONMENT,omitempty"`
 	SentryDSN     string                `json:"REACT_APP_SENTRY_DSN,omitempty"`
-	Teams         []string              `json:"REACT_APP_TEAMS"`
-	Streams       []string              `json:"REACT_APP_STREAMS"`
+	Streams       config.Streams        `json:"REACT_APP_STREAMS"`
 	Docs          config.Documentations `json:"REACT_APP_DOC_LINKS"`
-
-	config.UIConfig
 }
 
 func (h uiEnvHandler) handler(w http.ResponseWriter, r *http.Request) {