Sync issues, PRs, labels, milestones, comments from GitHub mirrors

cmd/admin.go

@@ -389,7 +389,8 @@ func runRepoSyncReleases(_ *cli.Context) error {
 			}
 			log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
 
-			if err = repo_module.SyncReleasesWithTags(repo, gitRepo); err != nil {
+			tagOnlyReleases := false
+			if err = repo_module.SyncReleasesWithTags(repo, gitRepo, tagOnlyReleases); err != nil {
 				log.Warn(" SyncReleasesWithTags: %v", err)
 				gitRepo.Close()
 				continue

models/issues/comment.go

@@ -225,6 +225,7 @@ type Comment struct {
 	Poster           *user_model.User `xorm:"-"`
 	OriginalAuthor   string
 	OriginalAuthorID int64
+	OriginalID       int64  // Only used in synced comments
 	IssueID          int64  `xorm:"INDEX"`
 	Issue            *Issue `xorm:"-"`
 	LabelID          int64

models/issues/label.go

@@ -91,6 +91,8 @@ type Label struct {
 	CreatedUnix     timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix     timeutil.TimeStamp `xorm:"INDEX updated"`
 
+	OriginalID int64 // Only for migrating data from other system, used for syncing
+
 	NumOpenIssues     int    `xorm:"-"`
 	NumOpenRepoIssues int64  `xorm:"-"`
 	IsChecked         bool   `xorm:"-"`
@@ -221,6 +223,10 @@ func NewLabels(labels ...*Label) error {
 
 // UpdateLabel updates label information.
 func UpdateLabel(l *Label) error {
+	return updateLabel(db.DefaultContext, l)
+}
+
+func updateLabel(ctx context.Context, l *Label) error {
 	color, err := label.NormalizeColor(l.Color)
 	if err != nil {
 		return err
@@ -232,19 +238,27 @@ func UpdateLabel(l *Label) error {
 
 // DeleteLabel delete a label
 func DeleteLabel(id, labelID int64) error {
-	l, err := GetLabelByID(db.DefaultContext, labelID)
+	ctx, committer, err := db.TxContext(db.DefaultContext)
 	if err != nil {
-		if IsErrLabelNotExist(err) {
-			return nil
-		}
 		return err
 	}
+	defer committer.Close()
 
-	ctx, committer, err := db.TxContext(db.DefaultContext)
+	if err = deleteLabel(ctx, id, labelID); err != nil {
+		return err
+	}
+
+	return committer.Commit()
+}
+
+func deleteLabel(ctx context.Context, id, labelID int64) error {
+	l, err := GetLabelByID(ctx, labelID)
 	if err != nil {
+		if IsErrLabelNotExist(err) {
+			return nil
+		}
 		return err
 	}
-	defer committer.Close()
 
 	sess := db.GetEngine(ctx)
 
@@ -257,7 +271,9 @@ func DeleteLabel(id, labelID int64) error {
 
 	if _, err = sess.ID(labelID).Delete(new(Label)); err != nil {
 		return err
-	} else if _, err = sess.
+	}
+
+	if _, err = sess.
 		Where("label_id = ?", labelID).
 		Delete(new(IssueLabel)); err != nil {
 		return err
@@ -268,7 +284,7 @@ func DeleteLabel(id, labelID int64) error {
 		return err
 	}
 
-	return committer.Commit()
+	return nil
 }
 
 // GetLabelByID returns a label by given ID.
@@ -396,6 +412,74 @@ func GetLabelsByRepoID(ctx context.Context, repoID int64, sortType string, listO
 	return labels, sess.Find(&labels)
 }
 
+// UpdateLabels adds, updates, and deletes relevant labels for the given repository.
+func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error {
+	ctx, committer, err := db.TxContext(db.DefaultContext)
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{})
+	if err != nil {
+		return err
+	}
+	labelsToAdd := make([]*Label, 0)
+	labelsToUpdate := make([]*Label, 0)
+	labelsToDelete := make([]*Label, 0)
+
+	for _, l := range labels {
+		var foundLabel *Label
+		for _, existingLabel := range existingLabels {
+			if existingLabel.OriginalID == l.OriginalID {
+				foundLabel = existingLabel
+				break
+			}
+		}
+
+		if foundLabel == nil {
+			labelsToAdd = append(labelsToAdd, l)
+		} else if foundLabel.Name != l.Name || foundLabel.Description != l.Description ||
+			foundLabel.Color != l.Color {
+			l.RepoID = repoID
+			labelsToUpdate = append(labelsToUpdate, l)
+		}
+	}
+
+	for _, existingLabel := range existingLabels {
+		found := false
+		for _, label := range labels {
+			if label.OriginalID == existingLabel.OriginalID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			labelsToDelete = append(labelsToDelete, existingLabel)
+		}
+	}
+
+	for _, l := range labelsToAdd {
+		if err = NewLabel(ctx, l); err != nil {
+			return err
+		}
+	}
+
+	for _, l := range labelsToUpdate {
+		if err = updateLabel(ctx, l); err != nil {
+			return err
+		}
+	}
+
+	for _, l := range labelsToDelete {
+		if err = deleteLabel(ctx, repoID, l.ID); err != nil {
+			return err
+		}
+	}
+
+	return committer.Commit()
+}
+
 // CountLabelsByRepoID count number of all labels that belong to given repository by ID.
 func CountLabelsByRepoID(repoID int64) (int64, error) {
 	return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repoID).Count(&Label{})

models/issues/milestone.go

@@ -64,6 +64,8 @@ type Milestone struct {
 	DeadlineString string `xorm:"-"`
 
 	TotalTrackedTime int64 `xorm:"-"`
+
+	OriginalID int64 // ID from the upstream syncing source
 }
 
 func init() {

models/issues/review.go

@@ -111,6 +111,7 @@ type Review struct {
 	ReviewerTeam     *organization.Team `xorm:"-"`
 	OriginalAuthor   string
 	OriginalAuthorID int64
+	OriginalID       int64
 	Issue            *Issue `xorm:"-"`
 	IssueID          int64  `xorm:"index"`
 	Content          string `xorm:"TEXT"`
@@ -531,17 +532,7 @@ func InsertReviews(reviews []*Review) error {
 			return err
 		}
 
-		if _, err := sess.NoAutoTime().Insert(&Comment{
-			Type:             CommentTypeReview,
-			Content:          review.Content,
-			PosterID:         review.ReviewerID,
-			OriginalAuthor:   review.OriginalAuthor,
-			OriginalAuthorID: review.OriginalAuthorID,
-			IssueID:          review.IssueID,
-			ReviewID:         review.ID,
-			CreatedUnix:      review.CreatedUnix,
-			UpdatedUnix:      review.UpdatedUnix,
-		}); err != nil {
+		if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil {
 			return err
 		}
 
@@ -559,6 +550,113 @@ func InsertReviews(reviews []*Review) error {
 	return committer.Commit()
 }
 
+// UpsertReviews inserts new reviews and updates existing ones.
+// This function is used for syncing from the pull mirror.
+func UpsertReviews(reviews []*Review) error {
+	ctx, committer, err := db.TxContext(db.DefaultContext)
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+	sess := db.GetEngine(ctx)
+
+	for _, review := range reviews {
+		exists, err := sess.Where("original_id = ?", review.OriginalID).Exist(&Review{})
+		if err != nil {
+			return err
+		}
+
+		if !exists {
+			if _, err := sess.NoAutoTime().Insert(review); err != nil {
+				return err
+			}
+
+			if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil {
+				return err
+			}
+
+			for _, c := range review.Comments {
+				c.ReviewID = review.ID
+			}
+
+			if len(review.Comments) > 0 {
+				if _, err := sess.NoAutoTime().Insert(review.Comments); err != nil {
+					return err
+				}
+			}
+		} else {
+			if _, err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Update(review); err != nil {
+				return err
+			}
+
+			// Get id of the review
+			if err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Find(review); err != nil {
+				return err
+			}
+
+			comment := generateCommentFromReview(review)
+			exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment)
+			if err != nil {
+				return err
+			}
+
+			if !exists {
+				if _, err := sess.NoAutoTime().Insert(comment); err != nil {
+					return err
+				}
+			} else {
+				if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil {
+					return err
+				}
+			}
+
+			for _, c := range review.Comments {
+				c.ReviewID = review.ID
+			}
+
+			if len(review.Comments) > 0 {
+				for _, comment := range review.Comments {
+					exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment)
+					if err != nil {
+						return err
+					}
+
+					if !exists {
+						if _, err := sess.NoAutoTime().Insert(comment); err != nil {
+							return err
+						}
+					} else {
+						if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return committer.Commit()
+}
+
+func existsCommentByReviewIDAndCreatedUnix(sess db.Engine, comment *Comment) (bool, error) {
+	return sess.Where("original_id = ?", comment.OriginalID).Exist(&Comment{})
+}
+
+func generateCommentFromReview(review *Review) *Comment {
+	return &Comment{
+		Type:             CommentTypeReview,
+		Content:          review.Content,
+		PosterID:         review.ReviewerID,
+		OriginalAuthor:   review.OriginalAuthor,
+		OriginalAuthorID: review.OriginalAuthorID,
+		IssueID:          review.IssueID,
+		ReviewID:         review.ID,
+		CreatedUnix:      review.CreatedUnix,
+		UpdatedUnix:      review.UpdatedUnix,
+		OriginalID:       review.OriginalID,
+	}
+}
+
 // AddReviewRequest add a review request from one reviewer
 func AddReviewRequest(ctx context.Context, issue *Issue, reviewer, doer *user_model.User) (*Comment, error) {
 	ctx, committer, err := db.TxContext(ctx)