feat(server): auto-update

Makefile

@@ -18,7 +18,7 @@ include rules.mk
 .PHONY: run
 run: install
 	time depviz --debug run --no-graph moul/depviz-test moul/depviz moul-bot/depviz-test moul/multipmuri moul/graphman
-	time depviz --debug server --without-recovery --godmode
+	time depviz --debug server --without-recovery --godmode --realm "DEV"
 
 .PHONY: update_examples
 update_examples:

cmd/depviz/main.go

@@ -21,6 +21,7 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"moul.io/depviz/internal/dvcore"
+	"moul.io/depviz/internal/dvparser"
 	"moul.io/depviz/internal/dvserver"
 	"moul.io/depviz/internal/dvstore"
 	"moul.io/godev"
@@ -48,12 +49,16 @@ var (
 	serverRequestTimeout     = serverFlags.Duration("request-timeout", 5*time.Second, "request timeout")
 	serverShutdownTimeout    = serverFlags.Duration("shutdowm-timeout", 6*time.Second, "shutdown timeout")
 	serverCORSAllowedOrigins = serverFlags.String("cors-allowed-origins", "*", "allowed CORS origins")
+	serverGitHubToken        = serverFlags.String("github-token", "", "GitHub token")
+	serverGitLabToken        = serverFlags.String("gitlab-token", "", "GitLab token")
+	serverNoAutoUpdate       = serverFlags.Bool("no-auto-update", false, "don't auto-update projects in background")
 	serverGodmode            = serverFlags.Bool("godmode", false, "enable dangerous API calls")
 	serverWithPprof          = serverFlags.Bool("with-pprof", false, "enable pprof endpoints")
 	serverWithoutRecovery    = serverFlags.Bool("without-recovery", false, "disable panic recovery (dev)")
 	serverWithoutCache       = serverFlags.Bool("without-cache", false, "disable HTTP caching")
 	serverAuth               = serverFlags.String("auth", "", "authentication password")
 	serverRealm              = serverFlags.String("realm", "DepViz", "server Realm")
+	serverAutoUpdateInterval = serverFlags.Duration("auto-update-interval", 2*time.Minute, "time between two auto-updates")
 
 	runFlags            = flag.NewFlagSet("run", flag.ExitOnError)
 	runNoPull           = runFlags.Bool("no-pull", false, "don't pull providers (graph only)")
@@ -257,8 +262,8 @@ func execRun(args []string) error {
 		NoGraph:          *runNoGraph,
 		NoPull:           *runNoPull,
 		Format:           *runFormat,
-		GitHubToken:      *runGitHubToken,
 		Resync:           *runResync,
+		GitHubToken:      *runGitHubToken,
 		GitLabToken:      *runGitLabToken,
 		ShowClosed:       *runShowClosed,
 		HideIsolated:     *runHideIsolated,
@@ -285,6 +290,11 @@ func execServer(args []string) error {
 			return fmt.Errorf("init store: %w", err)
 		}
 
+		targets, err := dvparser.ParseTargets(args)
+		if err != nil {
+			return fmt.Errorf("parse targets: %w", err)
+		}
+
 		opts := dvserver.Opts{
 			Logger:             logger,
 			HTTPBind:           *serverHTTPBind,
@@ -298,6 +308,11 @@ func execServer(args []string) error {
 			Auth:               *serverAuth,
 			Realm:              *serverRealm,
 			Godmode:            *serverGodmode,
+			GitHubToken:        *serverGitHubToken,
+			GitLabToken:        *serverGitLabToken,
+			NoAutoUpdate:       *serverNoAutoUpdate,
+			AutoUpdateTargets:  targets,
+			AutoUpdateInterval: *serverAutoUpdateInterval,
 		}
 		svc, err = dvserver.New(ctx, store, schemaConfig, opts)
 		if err != nil {

internal/dvcore/run.go

@@ -58,13 +58,8 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error {
 	}
 
 	if !opts.NoPull {
-		batches, err := pullBatches(targets, h, opts)
-		if err != nil {
-			return fmt.Errorf("pull batches: %w", err)
-		}
-		err = saveBatches(h, opts.Schema, batches)
-		if err != nil {
-			return fmt.Errorf("save batches: %w", err)
+		if err := PullAndSave(targets, h, opts.Schema, opts.GitHubToken, opts.GitLabToken, opts.Resync, opts.Logger); err != nil {
+			return fmt.Errorf("pull: %w", err)
 		}
 	}
 
@@ -156,7 +151,19 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error {
 	return nil
 }
 
-func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, opts RunOpts) ([]dvmodel.Batch, error) {
+func PullAndSave(targets []multipmuri.Entity, h *cayley.Handle, schema *schema.Config, githubToken string, gitlabToken string, resync bool, logger *zap.Logger) error {
+	batches, err := pullBatches(targets, h, githubToken, gitlabToken, resync, logger)
+	if err != nil {
+		return fmt.Errorf("pull batches: %w", err)
+	}
+	err = saveBatches(h, schema, batches)
+	if err != nil {
+		return fmt.Errorf("save batches: %w", err)
+	}
+	return nil
+}
+
+func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, githubToken string, gitlabToken string, resync bool, logger *zap.Logger) ([]dvmodel.Batch, error) {
 	// FIXME: handle the special '@me' target
 	var (
 		wg      sync.WaitGroup
@@ -175,22 +182,22 @@ func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, opts RunOpts) ([
 
 				ghOpts := githubprovider.Opts{
 					// FIXME: Since: lastUpdated,
-					Logger: opts.Logger.Named("github"),
+					Logger: logger.Named("github"),
 				}
 
-				if !opts.Resync {
+				if !resync {
 					since, err := dvstore.LastUpdatedIssueInRepo(ctx, h, repo)
 					if err != nil {
-						opts.Logger.Warn("failed to get last updated issue", zap.Error(err))
+						logger.Warn("failed to get last updated issue", zap.Error(err))
 
 					}
 					ghOpts.Since = &since
 				}
 
-				githubprovider.FetchRepo(ctx, repo, opts.GitHubToken, out, ghOpts)
+				githubprovider.FetchRepo(ctx, repo, githubToken, out, ghOpts)
 			}(target)
 		//case multipmuri.GitLabProvider:
-		//go gitlab.Pull(target, &wg, opts.GitlabToken, db, out)
+		//go gitlab.Pull(target, &wg, gitlabToken, db, out)
 		default:
 			// FIXME: clean context-based exit
 			panic(fmt.Sprintf("unsupported provider: %v", provider))

internal/dvserver/server.go

@@ -27,6 +27,8 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"moul.io/depviz/internal/chiutil"
+	"moul.io/depviz/internal/dvcore"
+	"moul.io/multipmuri"
 )
 
 type Opts struct {
@@ -42,6 +44,11 @@ type Opts struct {
 	WithoutCache       bool
 	Auth               string
 	Realm              string
+	GitHubToken        string
+	GitLabToken        string
+	NoAutoUpdate       bool
+	AutoUpdateTargets  []multipmuri.Entity
+	AutoUpdateInterval time.Duration
 }
 
 type Service interface {
@@ -81,6 +88,9 @@ func New(ctx context.Context, h *cayley.Handle, schema *schema.Config, opts Opts
 	if opts.Realm == "" {
 		opts.Realm = "DepViz"
 	}
+	if opts.AutoUpdateInterval == 0 {
+		opts.AutoUpdateInterval = 2 * time.Minute
+	}
 
 	svc := service{
 		ctx:    ctx,
@@ -236,11 +246,36 @@ func New(ctx context.Context, h *cayley.Handle, schema *schema.Config, opts Opts
 		})
 	}
 
+	if !opts.NoAutoUpdate && len(opts.AutoUpdateTargets) > 0 {
+		ctx, cancel := context.WithCancel(context.Background())
+
+		svc.workers.Add(func() error {
+			svc.recurringTask(opts.AutoUpdateTargets)
+			for {
+				select {
+				case <-ctx.Done():
+					return nil
+				case <-time.After(opts.AutoUpdateInterval):
+					svc.recurringTask(opts.AutoUpdateTargets)
+				}
+			}
+		}, func(error) {
+			cancel()
+		})
+	}
+
 	// FIXME: add grpc-web support?
 
 	return &svc, nil
 }
 
+func (s *service) recurringTask(targets []multipmuri.Entity) {
+	s.opts.Logger.Debug("pull and save", zap.Any("targets", targets))
+	if err := dvcore.PullAndSave(targets, s.h, s.schema, s.opts.GitHubToken, s.opts.GitLabToken, false, s.opts.Logger); err != nil {
+		s.opts.Logger.Warn("pull and save", zap.Error(err))
+	}
+}
+
 func (s *service) Run() error {
 	return s.workers.Run()
 }