From b37a66763cf454dfdc24347a4aaae7a5e10f194e Mon Sep 17 00:00:00 2001 From: vclemenzi Date: Mon, 4 Mar 2024 17:19:02 +0100 Subject: [PATCH 1/2] feat(performance): multi thread repo selection --- database/GetRepositories.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/database/GetRepositories.go b/database/GetRepositories.go index 2cb669e..88951d9 100644 --- a/database/GetRepositories.go +++ b/database/GetRepositories.go @@ -1,6 +1,8 @@ package database import ( + "sync" + "github.com/gitarchived/updater/git" "github.com/gitarchived/updater/models" "gorm.io/gorm" @@ -16,20 +18,33 @@ func GetRepositories(db *gorm.DB, host models.Host, force bool) ([]Repository, e var results []Repository data := db.Where("host = ? AND deleted = ?", host.Name, false).Find(&repositories) + limiter := make(chan int, 10) + wg := sync.WaitGroup{} + wg.Add(len(repositories)) for _, r := range repositories { - commit, err := git.GetLastCommit(r, host) - - if err != nil { - if err := db.Model(&models.Repository{}).Where("id = ?", r.ID).Update("deleted", true); err.Error != nil { - continue + limiter <- 1 + go func(r models.Repository) { + defer func() { + <-limiter + wg.Done() + }() + + commit, err := git.GetLastCommit(r, host) + + if err != nil { + if err := db.Model(&models.Repository{}).Where("id = ?", r.ID).Update("deleted", true); err.Error != nil { + data.Error = err.Error + } } - } - if commit != r.LastCommit || force { - results = append(results, Repository{Repository: r, NewCommitHash: commit}) - } + if commit != r.LastCommit || force { + results = append(results, Repository{Repository: r, NewCommitHash: commit}) + } + }(r) } + wg.Wait() + return results, data.Error } From 26ff15518be00a7405d0a28a1574b890c6e58b8f Mon Sep 17 00:00:00 2001 From: vclemenzi Date: Mon, 4 Mar 2024 17:28:38 +0100 Subject: [PATCH 2/2] fix: close chan --- database/GetRepositories.go | 1 + 1 file changed, 1 insertion(+) diff --git a/database/GetRepositories.go b/database/GetRepositories.go index 88951d9..fa0d74e 100644 --- a/database/GetRepositories.go +++ b/database/GetRepositories.go @@ -45,6 +45,7 @@ func GetRepositories(db *gorm.DB, host models.Host, force bool) ([]Repository, e } wg.Wait() + close(limiter) return results, data.Error }