diff --git a/Makefile b/Makefile index 8df4ac187..ff3796f70 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ include rules.mk .PHONY: run run: install time depviz --debug run --no-graph moul/depviz-test moul/depviz moul-bot/depviz-test moul/multipmuri moul/graphman - time depviz --debug server --without-recovery --godmode + time depviz --debug server --without-recovery --godmode --realm "DEV" .PHONY: update_examples update_examples: diff --git a/cmd/depviz/main.go b/cmd/depviz/main.go index cce8bfa56..09e22b6b2 100644 --- a/cmd/depviz/main.go +++ b/cmd/depviz/main.go @@ -21,6 +21,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" "moul.io/depviz/internal/dvcore" + "moul.io/depviz/internal/dvparser" "moul.io/depviz/internal/dvserver" "moul.io/depviz/internal/dvstore" "moul.io/godev" @@ -48,12 +49,16 @@ var ( serverRequestTimeout = serverFlags.Duration("request-timeout", 5*time.Second, "request timeout") serverShutdownTimeout = serverFlags.Duration("shutdowm-timeout", 6*time.Second, "shutdown timeout") serverCORSAllowedOrigins = serverFlags.String("cors-allowed-origins", "*", "allowed CORS origins") + serverGitHubToken = serverFlags.String("github-token", "", "GitHub token") + serverGitLabToken = serverFlags.String("gitlab-token", "", "GitLab token") + serverNoAutoUpdate = serverFlags.Bool("no-auto-update", false, "don't auto-update projects in background") serverGodmode = serverFlags.Bool("godmode", false, "enable dangerous API calls") serverWithPprof = serverFlags.Bool("with-pprof", false, "enable pprof endpoints") serverWithoutRecovery = serverFlags.Bool("without-recovery", false, "disable panic recovery (dev)") serverWithoutCache = serverFlags.Bool("without-cache", false, "disable HTTP caching") serverAuth = serverFlags.String("auth", "", "authentication password") serverRealm = serverFlags.String("realm", "DepViz", "server Realm") + serverAutoUpdateInterval = serverFlags.Duration("auto-update-interval", 2*time.Minute, "time between two auto-updates") runFlags = flag.NewFlagSet("run", flag.ExitOnError) runNoPull = runFlags.Bool("no-pull", false, "don't pull providers (graph only)") @@ -257,8 +262,8 @@ func execRun(args []string) error { NoGraph: *runNoGraph, NoPull: *runNoPull, Format: *runFormat, - GitHubToken: *runGitHubToken, Resync: *runResync, + GitHubToken: *runGitHubToken, GitLabToken: *runGitLabToken, ShowClosed: *runShowClosed, HideIsolated: *runHideIsolated, @@ -285,6 +290,11 @@ func execServer(args []string) error { return fmt.Errorf("init store: %w", err) } + targets, err := dvparser.ParseTargets(args) + if err != nil { + return fmt.Errorf("parse targets: %w", err) + } + opts := dvserver.Opts{ Logger: logger, HTTPBind: *serverHTTPBind, @@ -298,6 +308,11 @@ func execServer(args []string) error { Auth: *serverAuth, Realm: *serverRealm, Godmode: *serverGodmode, + GitHubToken: *serverGitHubToken, + GitLabToken: *serverGitLabToken, + NoAutoUpdate: *serverNoAutoUpdate, + AutoUpdateTargets: targets, + AutoUpdateInterval: *serverAutoUpdateInterval, } svc, err = dvserver.New(ctx, store, schemaConfig, opts) if err != nil { diff --git a/gen.sum b/gen.sum index 7f8e30c6e..611c75333 100644 --- a/gen.sum +++ b/gen.sum @@ -1,3 +1,3 @@ 4668890e2f9fc8852ff7696d493dab0d644fc5f4 ./api/dvmodel.proto -c8f0fbee003b669c1f736e7f6c9d43221ede90f7 Makefile +f425d5e6513db76befef3aaf77a19e53252450b4 Makefile f784b11d889417b3ce30d65744799a0e992e1201 ./api/dvserver.proto diff --git a/internal/dvcore/run.go b/internal/dvcore/run.go index 77aaa014c..3352eae8a 100644 --- a/internal/dvcore/run.go +++ b/internal/dvcore/run.go @@ -58,13 +58,8 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error { } if !opts.NoPull { - batches, err := pullBatches(targets, h, opts) - if err != nil { - return fmt.Errorf("pull batches: %w", err) - } - err = saveBatches(h, opts.Schema, batches) - if err != nil { - return fmt.Errorf("save batches: %w", err) + if err := PullAndSave(targets, h, opts.Schema, opts.GitHubToken, opts.GitLabToken, opts.Resync, opts.Logger); err != nil { + return fmt.Errorf("pull: %w", err) } } @@ -156,7 +151,19 @@ func Run(h *cayley.Handle, args []string, opts RunOpts) error { return nil } -func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, opts RunOpts) ([]dvmodel.Batch, error) { +func PullAndSave(targets []multipmuri.Entity, h *cayley.Handle, schema *schema.Config, githubToken string, gitlabToken string, resync bool, logger *zap.Logger) error { + batches, err := pullBatches(targets, h, githubToken, gitlabToken, resync, logger) + if err != nil { + return fmt.Errorf("pull batches: %w", err) + } + err = saveBatches(h, schema, batches) + if err != nil { + return fmt.Errorf("save batches: %w", err) + } + return nil +} + +func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, githubToken string, gitlabToken string, resync bool, logger *zap.Logger) ([]dvmodel.Batch, error) { // FIXME: handle the special '@me' target var ( wg sync.WaitGroup @@ -175,22 +182,22 @@ func pullBatches(targets []multipmuri.Entity, h *cayley.Handle, opts RunOpts) ([ ghOpts := githubprovider.Opts{ // FIXME: Since: lastUpdated, - Logger: opts.Logger.Named("github"), + Logger: logger.Named("github"), } - if !opts.Resync { + if !resync { since, err := dvstore.LastUpdatedIssueInRepo(ctx, h, repo) if err != nil { - opts.Logger.Warn("failed to get last updated issue", zap.Error(err)) + logger.Warn("failed to get last updated issue", zap.Error(err)) } ghOpts.Since = &since } - githubprovider.FetchRepo(ctx, repo, opts.GitHubToken, out, ghOpts) + githubprovider.FetchRepo(ctx, repo, githubToken, out, ghOpts) }(target) //case multipmuri.GitLabProvider: - //go gitlab.Pull(target, &wg, opts.GitlabToken, db, out) + //go gitlab.Pull(target, &wg, gitlabToken, db, out) default: // FIXME: clean context-based exit panic(fmt.Sprintf("unsupported provider: %v", provider)) diff --git a/internal/dvserver/server.go b/internal/dvserver/server.go index 2ad3fbd50..9e019f8bf 100644 --- a/internal/dvserver/server.go +++ b/internal/dvserver/server.go @@ -27,6 +27,8 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" "moul.io/depviz/internal/chiutil" + "moul.io/depviz/internal/dvcore" + "moul.io/multipmuri" ) type Opts struct { @@ -42,6 +44,11 @@ type Opts struct { WithoutCache bool Auth string Realm string + GitHubToken string + GitLabToken string + NoAutoUpdate bool + AutoUpdateTargets []multipmuri.Entity + AutoUpdateInterval time.Duration } type Service interface { @@ -81,6 +88,9 @@ func New(ctx context.Context, h *cayley.Handle, schema *schema.Config, opts Opts if opts.Realm == "" { opts.Realm = "DepViz" } + if opts.AutoUpdateInterval == 0 { + opts.AutoUpdateInterval = 2 * time.Minute + } svc := service{ ctx: ctx, @@ -236,11 +246,36 @@ func New(ctx context.Context, h *cayley.Handle, schema *schema.Config, opts Opts }) } + if !opts.NoAutoUpdate && len(opts.AutoUpdateTargets) > 0 { + ctx, cancel := context.WithCancel(context.Background()) + + svc.workers.Add(func() error { + svc.recurringTask(opts.AutoUpdateTargets) + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(opts.AutoUpdateInterval): + svc.recurringTask(opts.AutoUpdateTargets) + } + } + }, func(error) { + cancel() + }) + } + // FIXME: add grpc-web support? return &svc, nil } +func (s *service) recurringTask(targets []multipmuri.Entity) { + s.opts.Logger.Debug("pull and save", zap.Any("targets", targets)) + if err := dvcore.PullAndSave(targets, s.h, s.schema, s.opts.GitHubToken, s.opts.GitLabToken, false, s.opts.Logger); err != nil { + s.opts.Logger.Warn("pull and save", zap.Error(err)) + } +} + func (s *service) Run() error { return s.workers.Run() }