forked from Shiloh/githaven
Multiple tokens support for migrating from github (#17134)
* multiple tokens support for migrating from github * improve code and token description * Fix bug * Add comment for get client
This commit is contained in:
parent
56362043d3
commit
bdfd751af8
@ -69,12 +69,13 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
|
|||||||
type GithubDownloaderV3 struct {
|
type GithubDownloaderV3 struct {
|
||||||
base.NullDownloader
|
base.NullDownloader
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
client *github.Client
|
clients []*github.Client
|
||||||
repoOwner string
|
repoOwner string
|
||||||
repoName string
|
repoName string
|
||||||
userName string
|
userName string
|
||||||
password string
|
password string
|
||||||
rate *github.Rate
|
rates []*github.Rate
|
||||||
|
curClientIdx int
|
||||||
maxPerPage int
|
maxPerPage int
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,35 +90,69 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
|
|||||||
maxPerPage: 100,
|
maxPerPage: 100,
|
||||||
}
|
}
|
||||||
|
|
||||||
client := &http.Client{
|
if token != "" {
|
||||||
|
tokens := strings.Split(token, ",")
|
||||||
|
for _, token := range tokens {
|
||||||
|
token = strings.TrimSpace(token)
|
||||||
|
ts := oauth2.StaticTokenSource(
|
||||||
|
&oauth2.Token{AccessToken: token},
|
||||||
|
)
|
||||||
|
var client = &http.Client{
|
||||||
|
Transport: &oauth2.Transport{
|
||||||
|
Base: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: func(req *http.Request) (*url.URL, error) {
|
||||||
|
return proxy.Proxy()(req)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Source: oauth2.ReuseTokenSource(nil, ts),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
downloader.addClient(client, baseURL)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var client = &http.Client{
|
||||||
Transport: &http.Transport{
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
Proxy: func(req *http.Request) (*url.URL, error) {
|
||||||
req.SetBasicAuth(userName, password)
|
req.SetBasicAuth(userName, password)
|
||||||
return proxy.Proxy()(req)
|
return proxy.Proxy()(req)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if token != "" {
|
downloader.addClient(client, baseURL)
|
||||||
ts := oauth2.StaticTokenSource(
|
|
||||||
&oauth2.Token{AccessToken: token},
|
|
||||||
)
|
|
||||||
client = oauth2.NewClient(downloader.ctx, ts)
|
|
||||||
}
|
|
||||||
downloader.client = github.NewClient(client)
|
|
||||||
if baseURL != "https://github.com" {
|
|
||||||
downloader.client, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
|
|
||||||
}
|
}
|
||||||
return &downloader
|
return &downloader
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
|
||||||
|
githubClient := github.NewClient(client)
|
||||||
|
if baseURL != "https://github.com" {
|
||||||
|
githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
|
||||||
|
}
|
||||||
|
g.clients = append(g.clients, githubClient)
|
||||||
|
g.rates = append(g.rates, nil)
|
||||||
|
}
|
||||||
|
|
||||||
// SetContext set context
|
// SetContext set context
|
||||||
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
|
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
|
||||||
g.ctx = ctx
|
g.ctx = ctx
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *GithubDownloaderV3) sleep() {
|
func (g *GithubDownloaderV3) waitAndPickClient() {
|
||||||
for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining {
|
var recentIdx int
|
||||||
timer := time.NewTimer(time.Until(g.rate.Reset.Time))
|
var maxRemaining int
|
||||||
|
for i := 0; i < len(g.clients); i++ {
|
||||||
|
if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {
|
||||||
|
maxRemaining = g.rates[i].Remaining
|
||||||
|
recentIdx = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g.curClientIdx = recentIdx // if no max remain, it will always pick the first client.
|
||||||
|
|
||||||
|
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
|
||||||
|
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
|
||||||
select {
|
select {
|
||||||
case <-g.ctx.Done():
|
case <-g.ctx.Done():
|
||||||
util.StopTimer(timer)
|
util.StopTimer(timer)
|
||||||
@ -127,35 +162,43 @@ func (g *GithubDownloaderV3) sleep() {
|
|||||||
|
|
||||||
err := g.RefreshRate()
|
err := g.RefreshRate()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("g.client.RateLimits: %s", err)
|
log.Error("g.getClient().RateLimits: %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// RefreshRate update the current rate (doesn't count in rate limit)
|
// RefreshRate update the current rate (doesn't count in rate limit)
|
||||||
func (g *GithubDownloaderV3) RefreshRate() error {
|
func (g *GithubDownloaderV3) RefreshRate() error {
|
||||||
rates, _, err := g.client.RateLimits(g.ctx)
|
rates, _, err := g.getClient().RateLimits(g.ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// if rate limit is not enabled, ignore it
|
// if rate limit is not enabled, ignore it
|
||||||
if strings.Contains(err.Error(), "404") {
|
if strings.Contains(err.Error(), "404") {
|
||||||
g.rate = nil
|
g.setRate(nil)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
g.rate = rates.GetCore()
|
g.setRate(rates.GetCore())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (g *GithubDownloaderV3) getClient() *github.Client {
|
||||||
|
return g.clients[g.curClientIdx]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
|
||||||
|
g.rates[g.curClientIdx] = rate
|
||||||
|
}
|
||||||
|
|
||||||
// GetRepoInfo returns a repository information
|
// GetRepoInfo returns a repository information
|
||||||
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
|
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
|
|
||||||
// convert github repo to stand Repo
|
// convert github repo to stand Repo
|
||||||
return &base.Repository{
|
return &base.Repository{
|
||||||
@ -171,12 +214,12 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
|
|||||||
|
|
||||||
// GetTopics return github topics
|
// GetTopics return github topics
|
||||||
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
|
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
return r.Topics, nil
|
return r.Topics, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -185,8 +228,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
|
|||||||
var perPage = g.maxPerPage
|
var perPage = g.maxPerPage
|
||||||
var milestones = make([]*base.Milestone, 0, perPage)
|
var milestones = make([]*base.Milestone, 0, perPage)
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
|
ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
|
||||||
&github.MilestoneListOptions{
|
&github.MilestoneListOptions{
|
||||||
State: "all",
|
State: "all",
|
||||||
ListOptions: github.ListOptions{
|
ListOptions: github.ListOptions{
|
||||||
@ -196,7 +239,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
|
|
||||||
for _, m := range ms {
|
for _, m := range ms {
|
||||||
var state = "open"
|
var state = "open"
|
||||||
@ -233,8 +276,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
|
|||||||
var perPage = g.maxPerPage
|
var perPage = g.maxPerPage
|
||||||
var labels = make([]*base.Label, 0, perPage)
|
var labels = make([]*base.Label, 0, perPage)
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
|
ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
|
||||||
&github.ListOptions{
|
&github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: perPage,
|
PerPage: perPage,
|
||||||
@ -242,7 +285,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
|
|
||||||
for _, label := range ls {
|
for _, label := range ls {
|
||||||
labels = append(labels, convertGithubLabel(label))
|
labels = append(labels, convertGithubLabel(label))
|
||||||
@ -290,17 +333,17 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
|
|||||||
Created: asset.CreatedAt.Time,
|
Created: asset.CreatedAt.Time,
|
||||||
Updated: asset.UpdatedAt.Time,
|
Updated: asset.UpdatedAt.Time,
|
||||||
DownloadFunc: func() (io.ReadCloser, error) {
|
DownloadFunc: func() (io.ReadCloser, error) {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
asset, redirectURL, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
|
asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := g.RefreshRate(); err != nil {
|
if err := g.RefreshRate(); err != nil {
|
||||||
log.Error("g.client.RateLimits: %s", err)
|
log.Error("g.getClient().RateLimits: %s", err)
|
||||||
}
|
}
|
||||||
if asset == nil {
|
if asset == nil {
|
||||||
if redirectURL != "" {
|
if redirectURL != "" {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
|
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -308,7 +351,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
|
|||||||
resp, err := httpClient.Do(req)
|
resp, err := httpClient.Do(req)
|
||||||
err1 := g.RefreshRate()
|
err1 := g.RefreshRate()
|
||||||
if err1 != nil {
|
if err1 != nil {
|
||||||
log.Error("g.client.RateLimits: %s", err1)
|
log.Error("g.getClient().RateLimits: %s", err1)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -329,8 +372,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
|
|||||||
var perPage = g.maxPerPage
|
var perPage = g.maxPerPage
|
||||||
var releases = make([]*base.Release, 0, perPage)
|
var releases = make([]*base.Release, 0, perPage)
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
|
ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
|
||||||
&github.ListOptions{
|
&github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: perPage,
|
PerPage: perPage,
|
||||||
@ -338,7 +381,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
|
|
||||||
for _, release := range ls {
|
for _, release := range ls {
|
||||||
releases = append(releases, g.convertGithubRelease(release))
|
releases = append(releases, g.convertGithubRelease(release))
|
||||||
@ -366,13 +409,13 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
|
|||||||
}
|
}
|
||||||
|
|
||||||
var allIssues = make([]*base.Issue, 0, perPage)
|
var allIssues = make([]*base.Issue, 0, perPage)
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
|
issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, fmt.Errorf("error while listing repos: %v", err)
|
return nil, false, fmt.Errorf("error while listing repos: %v", err)
|
||||||
}
|
}
|
||||||
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
|
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
for _, issue := range issues {
|
for _, issue := range issues {
|
||||||
if issue.IsPullRequest() {
|
if issue.IsPullRequest() {
|
||||||
continue
|
continue
|
||||||
@ -386,15 +429,15 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
|
|||||||
// get reactions
|
// get reactions
|
||||||
var reactions []*base.Reaction
|
var reactions []*base.Reaction
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
|
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: perPage,
|
PerPage: perPage,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
if len(res) == 0 {
|
if len(res) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -464,25 +507,25 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
|
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error while listing repos: %v", err)
|
return nil, fmt.Errorf("error while listing repos: %v", err)
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
for _, comment := range comments {
|
for _, comment := range comments {
|
||||||
// get reactions
|
// get reactions
|
||||||
var reactions []*base.Reaction
|
var reactions []*base.Reaction
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
|
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: g.maxPerPage,
|
PerPage: g.maxPerPage,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
if len(res) == 0 {
|
if len(res) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -533,28 +576,28 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
|
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, fmt.Errorf("error while listing repos: %v", err)
|
return nil, false, fmt.Errorf("error while listing repos: %v", err)
|
||||||
}
|
}
|
||||||
var isEnd = resp.NextPage == 0
|
var isEnd = resp.NextPage == 0
|
||||||
|
|
||||||
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
|
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
for _, comment := range comments {
|
for _, comment := range comments {
|
||||||
// get reactions
|
// get reactions
|
||||||
var reactions []*base.Reaction
|
var reactions []*base.Reaction
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
|
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: g.maxPerPage,
|
PerPage: g.maxPerPage,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
if len(res) == 0 {
|
if len(res) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -598,13 +641,13 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
var allPRs = make([]*base.PullRequest, 0, perPage)
|
var allPRs = make([]*base.PullRequest, 0, perPage)
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
|
prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, fmt.Errorf("error while listing repos: %v", err)
|
return nil, false, fmt.Errorf("error while listing repos: %v", err)
|
||||||
}
|
}
|
||||||
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
|
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
for _, pr := range prs {
|
for _, pr := range prs {
|
||||||
var labels = make([]*base.Label, 0, len(pr.Labels))
|
var labels = make([]*base.Label, 0, len(pr.Labels))
|
||||||
for _, l := range pr.Labels {
|
for _, l := range pr.Labels {
|
||||||
@ -614,15 +657,15 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
|
|||||||
// get reactions
|
// get reactions
|
||||||
var reactions []*base.Reaction
|
var reactions []*base.Reaction
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
|
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: perPage,
|
PerPage: perPage,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
if len(res) == 0 {
|
if len(res) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -635,6 +678,9 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// download patch and saved as tmp file
|
||||||
|
g.waitAndPickClient()
|
||||||
|
|
||||||
allPRs = append(allPRs, &base.PullRequest{
|
allPRs = append(allPRs, &base.PullRequest{
|
||||||
Title: pr.GetTitle(),
|
Title: pr.GetTitle(),
|
||||||
Number: int64(pr.GetNumber()),
|
Number: int64(pr.GetNumber()),
|
||||||
@ -692,15 +738,15 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
|
|||||||
// get reactions
|
// get reactions
|
||||||
var reactions []*base.Reaction
|
var reactions []*base.Reaction
|
||||||
for i := 1; ; i++ {
|
for i := 1; ; i++ {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
|
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
|
||||||
Page: i,
|
Page: i,
|
||||||
PerPage: g.maxPerPage,
|
PerPage: g.maxPerPage,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
if len(res) == 0 {
|
if len(res) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -737,12 +783,12 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi
|
|||||||
PerPage: g.maxPerPage,
|
PerPage: g.maxPerPage,
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
|
reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error while listing repos: %v", err)
|
return nil, fmt.Errorf("error while listing repos: %v", err)
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
for _, review := range reviews {
|
for _, review := range reviews {
|
||||||
r := convertGithubReview(review)
|
r := convertGithubReview(review)
|
||||||
r.IssueIndex = context.LocalID()
|
r.IssueIndex = context.LocalID()
|
||||||
@ -751,12 +797,12 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi
|
|||||||
PerPage: g.maxPerPage,
|
PerPage: g.maxPerPage,
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
g.sleep()
|
g.waitAndPickClient()
|
||||||
reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
|
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error while listing repos: %v", err)
|
return nil, fmt.Errorf("error while listing repos: %v", err)
|
||||||
}
|
}
|
||||||
g.rate = &resp.Rate
|
g.setRate(&resp.Rate)
|
||||||
|
|
||||||
cs, err := g.convertGithubReviewComments(reviewComments)
|
cs, err := g.convertGithubReviewComments(reviewComments)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -887,6 +887,7 @@ migrate_items_releases = Releases
|
|||||||
migrate_repo = Migrate Repository
|
migrate_repo = Migrate Repository
|
||||||
migrate.clone_address = Migrate / Clone From URL
|
migrate.clone_address = Migrate / Clone From URL
|
||||||
migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repository
|
migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repository
|
||||||
|
migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit. WARN: Abusing this feature may violate the service provider's policy and lead to account blocking.
|
||||||
migrate.clone_local_path = or a local server path
|
migrate.clone_local_path = or a local server path
|
||||||
migrate.permission_denied = You are not allowed to import local repositories.
|
migrate.permission_denied = You are not allowed to import local repositories.
|
||||||
migrate.permission_denied_blocked = You are not allowed to import from blocked hosts.
|
migrate.permission_denied_blocked = You are not allowed to import from blocked hosts.
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
<label for="clone_addr">{{.i18n.Tr "repo.migrate.clone_address"}}</label>
|
<label for="clone_addr">{{.i18n.Tr "repo.migrate.clone_address"}}</label>
|
||||||
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
|
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
|
||||||
<span class="help">
|
<span class="help">
|
||||||
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
|
{{.i18n.Tr "repo.migrate.clone_address_desc"}}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@ -22,6 +22,9 @@
|
|||||||
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
|
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
|
||||||
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}>
|
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}>
|
||||||
<a target="_blank" href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a>
|
<a target="_blank" href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a>
|
||||||
|
<span class="help">
|
||||||
|
{{.i18n.Tr "repo.migrate.github_token_desc"}}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{{template "repo/migrate/options" .}}
|
{{template "repo/migrate/options" .}}
|
||||||
|
Loading…
Reference in New Issue
Block a user