Shiloh/githaven
Shiloh/githaven
1
0
Fork 1
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix context cache bug & enable context cache for dashabord commits' authors (#26991)

Browse Source 
Unfortunately, when a system setting hasn't been stored in the database,
it cannot be cached.
Meanwhile, this PR also uses context cache for push email avatar display
which should avoid to read user table via email address again and again.

According to my local test, this should reduce dashboard elapsed time
from 150ms -> 80ms .
This commit is contained in:
Lunny Xiao 2023-09-11 18:14:01 +08:00 committed by GitHub
parent 598465eb43
commit ebff0513db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 72 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
models/avatars/avatar.go
View File

@ -153,7 +153,12 @@ func generateEmailAvatarLink(ctx context.Context, email string, size int, final
return DefaultAvatarLink()
}
enableFederatedAvatar := system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureEnableFederatedAvatar)
disableGravatar := system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar,
setting.GetDefaultDisableGravatar(),
)
enableFederatedAvatar := system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureEnableFederatedAvatar,
setting.GetDefaultEnableFederatedAvatar(disableGravatar))
var err error
if enableFederatedAvatar && system_model.LibravatarService != nil {
@ -174,7 +179,6 @@ func generateEmailAvatarLink(ctx context.Context, email string, size int, final
return urlStr
}
disableGravatar := system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar)
if !disableGravatar {
// copy GravatarSourceURL, because we will modify its Path.
avatarURLCopy := *system_model.GravatarSourceURL

4
models/fixtures/system_setting.yml
View File

@ -1,6 +1,6 @@
-
id: 1
setting_key: 'disable_gravatar'
setting_key: 'picture.disable_gravatar'
setting_value: 'false'
version: 1
created: 1653533198
@ -8,7 +8,7 @@
-
id: 2
setting_key: 'enable_federated_avatar'
setting_key: 'picture.enable_federated_avatar'
setting_value: 'false'
version: 1
created: 1653533198

62
models/system/setting.go
View File

@ -94,11 +94,14 @@ func GetSetting(ctx context.Context, key string) (*Setting, error) {
const contextCacheKey = "system_setting"
// GetSettingWithCache returns the setting value via the key
func GetSettingWithCache(ctx context.Context, key string) (string, error) {
func GetSettingWithCache(ctx context.Context, key, defaultVal string) (string, error) {
return cache.GetWithContextCache(ctx, contextCacheKey, key, func() (string, error) {
return cache.GetString(genSettingCacheKey(key), func() (string, error) {
res, err := GetSetting(ctx, key)
if err != nil {
if IsErrSettingIsNotExist(err) {
return defaultVal, nil
}
return "", err
}
return res.SettingValue, nil
@ -108,17 +111,21 @@ func GetSettingWithCache(ctx context.Context, key string) (string, error) {
// GetSettingBool return bool value of setting,
// none existing keys and errors are ignored and result in false
func GetSettingBool(ctx context.Context, key string) bool {
s, _ := GetSetting(ctx, key)
if s == nil {
return false
}
func GetSettingBool(ctx context.Context, key string, defaultVal bool) (bool, error) {
s, err := GetSetting(ctx, key)
switch {
case err == nil:
v, _ := strconv.ParseBool(s.SettingValue)
return v
return v, nil
case IsErrSettingIsNotExist(err):
return defaultVal, nil
default:
return false, err
}
}
func GetSettingWithCacheBool(ctx context.Context, key string) bool {
s, _ := GetSettingWithCache(ctx, key)
func GetSettingWithCacheBool(ctx context.Context, key string, defaultVal bool) bool {
s, _ := GetSettingWithCache(ctx, key, strconv.FormatBool(defaultVal))
v, _ := strconv.ParseBool(s)
return v
}
@ -259,52 +266,41 @@ var (
)
func Init(ctx context.Context) error {
var disableGravatar bool
disableGravatarSetting, err := GetSetting(ctx, KeyPictureDisableGravatar)
if IsErrSettingIsNotExist(err) {
disableGravatar = setting_module.GetDefaultDisableGravatar()
disableGravatarSetting = &Setting{SettingValue: strconv.FormatBool(disableGravatar)}
} else if err != nil {
disableGravatar, err := GetSettingBool(ctx, KeyPictureDisableGravatar, setting_module.GetDefaultDisableGravatar())
if err != nil {
return err
} else {
disableGravatar = disableGravatarSetting.GetValueBool()
}
var enableFederatedAvatar bool
enableFederatedAvatarSetting, err := GetSetting(ctx, KeyPictureEnableFederatedAvatar)
if IsErrSettingIsNotExist(err) {
enableFederatedAvatar = setting_module.GetDefaultEnableFederatedAvatar(disableGravatar)
enableFederatedAvatarSetting = &Setting{SettingValue: strconv.FormatBool(enableFederatedAvatar)}
} else if err != nil {
enableFederatedAvatar, err := GetSettingBool(ctx, KeyPictureEnableFederatedAvatar, setting_module.GetDefaultEnableFederatedAvatar(disableGravatar))
if err != nil {
return err
} else {
enableFederatedAvatar = disableGravatarSetting.GetValueBool()
}
if setting_module.OfflineMode {
disableGravatar = true
enableFederatedAvatar = false
if !GetSettingBool(ctx, KeyPictureDisableGravatar) {
if !disableGravatar {
if err := SetSettingNoVersion(ctx, KeyPictureDisableGravatar, "true"); err != nil {
return fmt.Errorf("Failed to set setting %q: %w", KeyPictureDisableGravatar, err)
return fmt.Errorf("failed to set setting %q: %w", KeyPictureDisableGravatar, err)
}
}
if GetSettingBool(ctx, KeyPictureEnableFederatedAvatar) {
disableGravatar = true
if enableFederatedAvatar {
if err := SetSettingNoVersion(ctx, KeyPictureEnableFederatedAvatar, "false"); err != nil {
return fmt.Errorf("Failed to set setting %q: %w", KeyPictureEnableFederatedAvatar, err)
return fmt.Errorf("failed to set setting %q: %w", KeyPictureEnableFederatedAvatar, err)
}
}
enableFederatedAvatar = false
}
if enableFederatedAvatar || !disableGravatar {
var err error
GravatarSourceURL, err = url.Parse(setting_module.GravatarSource)
if err != nil {
return fmt.Errorf("Failed to parse Gravatar URL(%s): %w", setting_module.GravatarSource, err)
return fmt.Errorf("failed to parse Gravatar URL(%s): %w", setting_module.GravatarSource, err)
}
}
if GravatarSourceURL != nil && enableFederatedAvatarSetting.GetValueBool() {
if GravatarSourceURL != nil && enableFederatedAvatar {
LibravatarService = libravatar.New()
if GravatarSourceURL.Scheme == "https" {
LibravatarService.SetUseHTTPS(true)

4
models/user/avatar.go
View File

@ -67,7 +67,9 @@ func (u *User) AvatarLinkWithSize(ctx context.Context, size int) string {
useLocalAvatar := false
autoGenerateAvatar := false
disableGravatar := system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar)
disableGravatar := system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar,
setting.GetDefaultDisableGravatar(),
)
switch {
case u.UseCustomAvatar:

58
modules/repository/commits.go
View File

@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models/avatars"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@ -34,42 +35,36 @@ type PushCommits struct {
HeadCommit *PushCommit
CompareURL string
Len int
avatars map[string]string
emailUsers map[string]*user_model.User
}
// NewPushCommits creates a new PushCommits object.
func NewPushCommits() *PushCommits {
return &PushCommits{
avatars: make(map[string]string),
emailUsers: make(map[string]*user_model.User),
}
return &PushCommits{}
}
// toAPIPayloadCommit converts a single PushCommit to an api.PayloadCommit object.
func (pc *PushCommits) toAPIPayloadCommit(ctx context.Context, repoPath, repoLink string, commit *PushCommit) (*api.PayloadCommit, error) {
func (pc *PushCommits) toAPIPayloadCommit(ctx context.Context, emailUsers map[string]*user_model.User, repoPath, repoLink string, commit *PushCommit) (*api.PayloadCommit, error) {
var err error
authorUsername := ""
author, ok := pc.emailUsers[commit.AuthorEmail]
author, ok := emailUsers[commit.AuthorEmail]
if !ok {
author, err = user_model.GetUserByEmail(ctx, commit.AuthorEmail)
if err == nil {
authorUsername = author.Name
pc.emailUsers[commit.AuthorEmail] = author
emailUsers[commit.AuthorEmail] = author
}
} else {
authorUsername = author.Name
}
committerUsername := ""
committer, ok := pc.emailUsers[commit.CommitterEmail]
committer, ok := emailUsers[commit.CommitterEmail]
if !ok {
committer, err = user_model.GetUserByEmail(ctx, commit.CommitterEmail)
if err == nil {
// TODO: check errors other than email not found.
committerUsername = committer.Name
pc.emailUsers[commit.CommitterEmail] = committer
emailUsers[commit.CommitterEmail] = committer
}
} else {
committerUsername = committer.Name
@ -107,11 +102,10 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi
commits := make([]*api.PayloadCommit, len(pc.Commits))
var headCommit *api.PayloadCommit
if pc.emailUsers == nil {
pc.emailUsers = make(map[string]*user_model.User)
}
emailUsers := make(map[string]*user_model.User)
for i, commit := range pc.Commits {
apiCommit, err := pc.toAPIPayloadCommit(ctx, repoPath, repoLink, commit)
apiCommit, err := pc.toAPIPayloadCommit(ctx, emailUsers, repoPath, repoLink, commit)
if err != nil {
return nil, nil, err
}
@ -123,7 +117,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi
}
if pc.HeadCommit != nil && headCommit == nil {
var err error
headCommit, err = pc.toAPIPayloadCommit(ctx, repoPath, repoLink, pc.HeadCommit)
headCommit, err = pc.toAPIPayloadCommit(ctx, emailUsers, repoPath, repoLink, pc.HeadCommit)
if err != nil {
return nil, nil, err
}
@ -134,35 +128,21 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repoPath, repoLi
// AvatarLink tries to match user in database with e-mail
// in order to show custom avatar, and falls back to general avatar link.
func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string {
if pc.avatars == nil {
pc.avatars = make(map[string]string)
}
avatar, ok := pc.avatars[email]
if ok {
return avatar
}
size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor
u, ok := pc.emailUsers[email]
if !ok {
var err error
u, err = user_model.GetUserByEmail(ctx, email)
v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) {
u, err := user_model.GetUserByEmail(ctx, email)
if err != nil {
pc.avatars[email] = avatars.GenerateEmailAvatarFastLink(ctx, email, size)
if !user_model.IsErrUserNotExist(err) {
log.Error("GetUserByEmail: %v", err)
return ""
return "", err
}
} else {
pc.emailUsers[email] = u
}
}
if u != nil {
pc.avatars[email] = u.AvatarLinkWithSize(ctx, size)
return avatars.GenerateEmailAvatarFastLink(ctx, email, size), nil
}
return u.AvatarLinkWithSize(ctx, size), nil
})
return pc.avatars[email]
return v
}
// CommitToPushCommit transforms a git.Commit to PushCommit type.
@ -189,7 +169,5 @@ func GitToPushCommits(gitCommits []*git.Commit) *PushCommits {
HeadCommit: nil,
CompareURL: "",
Len: len(commits),
avatars: make(map[string]string),
emailUsers: make(map[string]*user_model.User),
}
}

8
modules/repository/commits_test.go
View File

@ -103,11 +103,9 @@ func TestPushCommits_ToAPIPayloadCommits(t *testing.T) {
assert.EqualValues(t, []string{"readme.md"}, headCommit.Modified)
}
func enableGravatar(t *testing.T) {
err := system_model.SetSettingNoVersion(db.DefaultContext, system_model.KeyPictureDisableGravatar, "false")
assert.NoError(t, err)
func initGravatarSource(t *testing.T) {
setting.GravatarSource = "https://secure.gravatar.com/avatar"
err = system_model.Init(db.DefaultContext)
err := system_model.Init(db.DefaultContext)
assert.NoError(t, err)
}
@ -134,7 +132,7 @@ func TestPushCommits_AvatarLink(t *testing.T) {
},
}
enableGravatar(t)
initGravatarSource(t)
assert.Equal(t,
"https://secure.gravatar.com/avatar/ab53a2911ddf9b4817ac01ddcd3d975f?d=identicon&s="+strconv.Itoa(28*setting.Avatar.RenderedSizeFactor),

7
routers/web/admin/users.go
View File

@ -314,7 +314,9 @@ func EditUser(ctx *context.Context) {
ctx.Data["DisableRegularOrgCreation"] = setting.Admin.DisableRegularOrgCreation
ctx.Data["DisableMigrations"] = setting.Repository.DisableMigrations
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar)
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar,
setting.GetDefaultDisableGravatar(),
)
prepareUserInfo(ctx)
if ctx.Written() {
@ -331,7 +333,8 @@ func EditUserPost(ctx *context.Context) {
ctx.Data["PageIsAdminUsers"] = true
ctx.Data["DisableMigrations"] = setting.Repository.DisableMigrations
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar)
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar,
setting.GetDefaultDisableGravatar())
u := prepareUserInfo(ctx)
if ctx.Written() {

8
routers/web/user/setting/profile.go
View File

@ -44,7 +44,9 @@ func Profile(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("settings.profile")
ctx.Data["PageIsSettingsProfile"] = true
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar)
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar,
setting.GetDefaultDisableGravatar(),
)
ctx.HTML(http.StatusOK, tplSettingsProfile)
}
@ -86,7 +88,9 @@ func ProfilePost(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("settings")
ctx.Data["PageIsSettingsProfile"] = true
ctx.Data["AllowedUserVisibilityModes"] = setting.Service.AllowedUserVisibilityModesSlice.ToVisibleTypeSlice()
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar)
ctx.Data["DisableGravatar"] = system_model.GetSettingWithCacheBool(ctx, system_model.KeyPictureDisableGravatar,
setting.GetDefaultDisableGravatar(),
)
if ctx.HasError() {
ctx.HTML(http.StatusOK, tplSettingsProfile)