linden123/githaven-fork
1
1
Fork 0
forked from Shiloh/githaven
Code Pull Requests Activity

Asynchronously populate the repo indexer (#3366)

Browse Source 
* Populate repo indexer in background

* Check if no repos exist

* race cond
This commit is contained in:
Ethan Koenig 2018-01-14 09:34:41 -08:00 committed by Lauris BH
parent a8325dd1cb
commit ccdb94992d
1 changed files with 41 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
models/repo_indexer.go
View File

@ -70,34 +70,58 @@ func InitRepoIndexer() {
if !setting.Indexer.RepoIndexerEnabled {
return
}
indexer.InitRepoIndexer(populateRepoIndexer)
repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
go processRepoIndexerOperationQueue()
}
// populateRepoIndexer populate the repo indexer with data
func populateRepoIndexer() error {
log.Info("Populating repository indexer (this may take a while)")
for page := 1; ; page++ {
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
Page: page,
PageSize: RepositoryListDefaultPageSize,
OrderBy: SearchOrderByID,
Private: true,
})
// populateRepoIndexerAsynchronously asynchronously populates the repo indexer
// with pre-existing data. This should only be run when the indexer is created
// for the first time.
func populateRepoIndexerAsynchronously() error {
exist, err := x.Table("repository").Exist()
if err != nil {
return err
} else if len(repos) == 0 {
} else if !exist {
return nil
}
var maxRepoID int64
if _, err = x.Select("MAX(id)").Table("repository").Get(&maxRepoID); err != nil {
return err
}
go populateRepoIndexer(maxRepoID)
return nil
}
// populateRepoIndexer populate the repo indexer with pre-existing data. This
// should only be run when the indexer is created for the first time.
func populateRepoIndexer(maxRepoID int64) {
log.Info("Populating the repo indexer with existing repositories")
// start with the maximum existing repo ID and work backwards, so that we
// don't include repos that are created after gitea starts; such repos will
// already be added to the indexer, and we don't need to add them again.
for maxRepoID > 0 {
repos := make([]*Repository, 0, RepositoryListDefaultPageSize)
err := x.Where("id <= ?", maxRepoID).
OrderBy("id DESC").
Limit(RepositoryListDefaultPageSize).
Find(&repos)
if err != nil {
log.Error(4, "populateRepoIndexer: %v", err)
return
} else if len(repos) == 0 {
break
}
for _, repo := range repos {
if err = updateRepoIndexer(repo); err != nil {
// only log error, since this should not prevent
// gitea from starting up
log.Error(4, "updateRepoIndexer: repoID=%d, %v", repo.ID, err)
}
repoIndexerOperationQueue <- repoIndexerOperation{
repo: repo,
deleted: false,
}
maxRepoID = repo.ID - 1
}
}
log.Info("Done populating the repo indexer with existing repositories")
}
func updateRepoIndexer(repo *Repository) error {