Add new contributors timeline (#199)

* Add new contributors timeline

* Added UI

go.mod

@@ -6,7 +6,7 @@ toolchain go1.23.5
 
 require (
 	github.com/Code-Hex/go-generics-cache v1.5.1
-	github.com/emanuelef/github-repo-activity-stats v0.2.45
+	github.com/emanuelef/github-repo-activity-stats v0.2.46
 	github.com/gofiber/contrib/otelfiber v1.0.10
 	github.com/gofiber/fiber/v2 v2.52.6
 	github.com/joho/godotenv v1.5.1

go.sum

@@ -12,8 +12,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emanuelef/github-repo-activity-stats v0.2.45 h1:YXib4OhnFkghcAkBEDy5VT77guzInccp0ehoTnvwP70=
-github.com/emanuelef/github-repo-activity-stats v0.2.45/go.mod h1:tXq0+KqbCDzEMnzBbnpl0CPFhorJAG9gQGgXZjB3Wfo=
+github.com/emanuelef/github-repo-activity-stats v0.2.46 h1:PhHXwWRrnb1L+sj60EHBegqU/erypJiPucQtY4YsH1A=
+github.com/emanuelef/github-repo-activity-stats v0.2.46/go.mod h1:w4SCCdW1qnzcADFCclSotfY0fYqwmYjB+xnyX9h17EQ=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=

main.go

@@ -70,6 +70,10 @@ type CommitsWithStatsResponse struct {
 	DefaultBranch string                `json:"defaultBranch"`
 }
 
+type ContributorsWithStatsResponse struct {
+	Contributors []stats.NewContributorsPerDay `json:"contributors"`
+}
+
 func getEnv(key, fallback string) string {
 	value, exists := os.LookupEnv(key)
 	if !exists {
@@ -124,6 +128,7 @@ func main() {
 	cacheForks := cache.New[string, ForksWithStatsResponse]()
 	cachePRs := cache.New[string, PRsWithStatsResponse]()
 	cacheCommits := cache.New[string, CommitsWithStatsResponse]()
+	cacheContributors := cache.New[string, ContributorsWithStatsResponse]()
 
 	cacheHackerNews := cache.New[string, []news.Article]()
 	cacheReddit := cache.New[string, []news.ArticleData]()
@@ -134,6 +139,7 @@ func main() {
 	onGoingForks := make(map[string]bool)
 	onGoingPRs := make(map[string]bool)
 	onGoingCommits := make(map[string]bool)
+	onGoingContributors := make(map[string]bool)
 
 	ghStatClients := make(map[string]*repostats.ClientGQL)
 
@@ -901,6 +907,110 @@ func main() {
 		return c.JSON(res)
 	})
 
+	app.Get("/allContributors", func(c *fiber.Ctx) error {
+		param := c.Query("repo")
+		randomIndex := rand.Intn(len(maps.Keys(ghStatClients)))
+		clientKey := c.Query("client", maps.Keys(ghStatClients)[randomIndex])
+		forceRefetch := c.Query("forceRefetch", "false") == "true"
+
+		client, ok := ghStatClients[clientKey]
+		if !ok {
+			return c.Status(404).SendString("Resource not found")
+		}
+
+		repo, err := url.QueryUnescape(param)
+		if err != nil {
+			return err
+		}
+
+		// needed because c.Query cannot be used as a map key
+		repo = fmt.Sprintf("%s", repo)
+		repo = strings.ToLower(repo)
+
+		ip := c.Get("X-Forwarded-For")
+
+		// If X-Forwarded-For is empty, fallback to RemoteIP
+		if ip == "" {
+			ip = c.IP()
+		}
+
+		userAgent := c.Get("User-Agent")
+		log.Printf("Contributors Request from IP: %s, Repo: %s User-Agent: %s\n", ip, repo, userAgent)
+
+		if strings.Contains(userAgent, "python-requests") {
+			return c.Status(404).SendString("Custom 404 Error: Resource not found")
+		}
+
+		span := trace.SpanFromContext(c.UserContext())
+		span.SetAttributes(attribute.String("github.repo", repo))
+		span.SetAttributes(attribute.String("caller.ip", ip))
+
+		if forceRefetch {
+			cacheContributors.Delete(repo)
+		}
+
+		if res, hit := cacheContributors.Get(repo); hit {
+			return c.JSON(res)
+		}
+
+		// if another request is already getting the data, skip and rely on SSE updates
+		if _, hit := onGoingContributors[repo]; hit {
+			return c.SendStatus(fiber.StatusNoContent)
+		}
+
+		onGoingContributors[repo] = true
+
+		updateChannel := make(chan int)
+		var allContributors []stats.NewContributorsPerDay
+
+		eg, ctx := errgroup.WithContext(ctx)
+
+		eg.Go(func() error {
+			allContributors, err = client.GetNewContributorsHistory(ctx, repo, updateChannel)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+		for progress := range updateChannel {
+			// fmt.Printf("Progress: %d\n", progress)
+
+			wg := &sync.WaitGroup{}
+
+			for _, s := range currentSessions.Sessions {
+				wg.Add(1)
+				go func(cs *session.Session) {
+					defer wg.Done()
+					if cs.Repo == repo {
+						cs.StateChannel <- progress
+					}
+				}(s)
+			}
+			wg.Wait()
+		}
+
+		if err := eg.Wait(); err != nil {
+			delete(onGoingContributors, repo)
+			return err
+		}
+
+		// defer close(updateChannel)
+
+		res := ContributorsWithStatsResponse{
+			Contributors: allContributors,
+		}
+
+		now := time.Now()
+		nextDay := now.UTC().Truncate(24 * time.Hour).Add(DAY_CACHED * 24 * time.Hour)
+		durationUntilEndOfDay := nextDay.Sub(now)
+
+		cacheContributors.Set(repo, res, cache.WithExpiration(durationUntilEndOfDay))
+		delete(onGoingContributors, repo)
+
+		return c.JSON(res)
+	})
+
 	app.Get("/limits", func(c *fiber.Ctx) error {
 		client, ok := ghStatClients["PAT"]
 		if !ok {

scripts/get_repos_data.sh

@@ -19,8 +19,4 @@ curl -s "http://143.47.226.125:8080/allCommits?repo=helm/helm" > /dev/null
 curl -s "http://143.47.226.125:8080/allPRs?repo=helm/helm" > /dev/null
 curl -s "http://143.47.226.125:8080/allIssues?repo=helm/helm" > /dev/null
 curl -s "http://143.47.226.125:8080/allForks?repo=helm/helm" > /dev/null
-sleep 600
-curl -s "http://143.47.226.125:8080/allCommits?repo=pyenv/pyenv" > /dev/null
-curl -s "http://143.47.226.125:8080/allPRs?repo=pyenv/pyenv" > /dev/null
-curl -s "http://143.47.226.125:8080/allIssues?repo=pyenv/pyenv" > /dev/null
-curl -s "http://143.47.226.125:8080/allForks?repo=pyenv/pyenv" > /dev/null
+curl -s "http://143.47.226.125:8080/allContributors?repo=helm/helm" > /dev/null

website/package.json

@@ -30,7 +30,7 @@
     "react-github-btn": "^1.4.0",
     "react-markdown": "^9.0.3",
     "react-pro-sidebar": "^1.1.0",
-    "react-router-dom": "^7.1.2",
+    "react-router-dom": "^7.1.3",
     "react-toastify": "^11.0.3",
     "react18-json-view": "^0.2.8"
   },

website/src/App.tsx

@@ -8,6 +8,7 @@ import IssuesTimeSeriesChart from "./IssuesTimeSeriesChart";
 import PRsTimeSeriesChart from "./PRsTimeSeriesChart";
 import ForksTimeSeriesChart from "./ForksTimeSeriesChart";
 import CommitsTimeSeriesChart from "./CommitsTimeSeriesChart";
+import ContributorsTimeSeriesChart from "./ContributorsTimeSeriesChart";
 import InfoPage from "./InfoPage";
 
 import { Sidebar, Menu, MenuItem } from "react-pro-sidebar";
@@ -22,6 +23,7 @@ import BugReportRoundedIcon from "@mui/icons-material/BugReportRounded";
 import AltRouteOutlinedIcon from "@mui/icons-material/AltRouteOutlined";
 import CallMergeRoundedIcon from '@mui/icons-material/CallMergeRounded';
 import CommitRoundedIcon from '@mui/icons-material/CommitRounded';
+import Diversity3OutlinedIcon from '@mui/icons-material/Diversity3Outlined';
 
 import { ThemeProvider, createTheme } from "@mui/material/styles";
 import CssBaseline from "@mui/material/CssBaseline";
@@ -87,7 +89,8 @@ function App() {
                   useLocation().pathname.includes("/issues") ||
                   useLocation().pathname.includes("/forks") ||
                   useLocation().pathname.includes("/prs") ||
-                  useLocation().pathname.includes("/commits")
+                  useLocation().pathname.includes("/commits") ||
+                  useLocation().pathname.includes("/contributors")
                 )
               }
             >
@@ -148,6 +151,17 @@ function App() {
             >
               Forks
             </MenuItem>
+            <MenuItem
+              component={<Link to="/contributors" className="link" />}
+              icon={
+                <Tooltip title="Contributors Timeline" placement="right">
+                  <Diversity3OutlinedIcon />
+                </Tooltip>
+              }
+              active={useLocation().pathname.includes("/contributors")}
+            >
+              Contributors
+            </MenuItem>
             <MenuItem
               component={<Link to="/table" className="link" />}
               icon={
@@ -192,6 +206,8 @@ function App() {
             <Route path="/prs/:user/:repository" element={<PRsTimeSeriesChart />} />
             <Route path="/commits" element={<CommitsTimeSeriesChart />} />
             <Route path="/commits/:user/:repository" element={<CommitsTimeSeriesChart />} />
+            <Route path="/contributors" element={<ContributorsTimeSeriesChart />} />
+            <Route path="/contributors/:user/:repository" element={<ContributorsTimeSeriesChart />} />
           </Routes>
         </section>
       </div>