linden123/githaven-fork
1
1
Fork 0
forked from Shiloh/githaven
Code Pull Requests Activity

Refactor cache and disable go-chi cache (#30417)

Browse Source 
use built-in cache package to wrap external go-chi cache package
This commit is contained in:
wxiaoguang 2024-04-13 16:38:44 +08:00 committed by GitHub
parent 8fd8978b49
commit c248f010ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 198 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.golangci.yml
View File

@ -86,6 +86,8 @@ linters-settings:
desc: do not use the internal package, use AddXxx function instead
- pkg: gopkg.in/ini.v1
desc: do not use the ini package, use gitea's config system instead
- pkg: gitea.com/go-chi/cache
desc: do not use the go-chi cache package, use gitea's cache system
issues:
max-issues-per-linter: 0

138
modules/cache/cache.go vendored
View File

@ -4,149 +4,75 @@
package cache
import (
"fmt"
"strconv"
"time"
"code.gitea.io/gitea/modules/setting"
mc "gitea.com/go-chi/cache"
_ "gitea.com/go-chi/cache/memcache" // memcache plugin for cache
)
var conn mc.Cache
func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
return mc.NewCacher(mc.Options{
Adapter: cacheConfig.Adapter,
AdapterConfig: cacheConfig.Conn,
Interval: cacheConfig.Interval,
})
}
var defaultCache StringCache
// Init start cache service
func Init() error {
var err error
if conn == nil {
if conn, err = newCache(setting.CacheService.Cache); err != nil {
if defaultCache == nil {
c, err := NewStringCache(setting.CacheService.Cache)
if err != nil {
return err
}
if err = conn.Ping(); err != nil {
for i := 0; i < 10; i++ {
if err = c.Ping(); err == nil {
break
}
time.Sleep(time.Second)
}
if err != nil {
return err
}
defaultCache = c
}
return err
return nil
}
// GetCache returns the currently configured cache
func GetCache() mc.Cache {
return conn
func GetCache() StringCache {
return defaultCache
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
if conn == nil || setting.CacheService.TTL == 0 {
if defaultCache == nil || setting.CacheService.TTL == 0 {
return getFunc()
}
cached := conn.Get(key)
if cached == nil {
cached, exist := defaultCache.Get(key)
if !exist {
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
if value, ok := cached.(string); ok {
return value, nil
}
if stringer, ok := cached.(fmt.Stringer); ok {
return stringer.String(), nil
}
return fmt.Sprintf("%s", cached), nil
}
// GetInt returns key value from cache with callback when no key exists in cache
func GetInt(key string, getFunc func() (int, error)) (int, error) {
if conn == nil || setting.CacheService.TTL == 0 {
return getFunc()
}
cached := conn.Get(key)
if cached == nil {
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
switch v := cached.(type) {
case int:
return v, nil
case string:
value, err := strconv.Atoi(v)
if err != nil {
return 0, err
}
return value, nil
default:
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
return value, defaultCache.Put(key, value, setting.CacheService.TTLSeconds())
}
return cached, nil
}
// GetInt64 returns key value from cache with callback when no key exists in cache
func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
if conn == nil || setting.CacheService.TTL == 0 {
return getFunc()
s, err := GetString(key, func() (string, error) {
v, err := getFunc()
return strconv.FormatInt(v, 10), err
})
if err != nil {
return 0, err
}
cached := conn.Get(key)
if cached == nil {
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
switch v := conn.Get(key).(type) {
case int64:
return v, nil
case string:
value, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return 0, err
}
return value, nil
default:
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
if s == "" {
return 0, nil
}
return strconv.ParseInt(s, 10, 64)
}
// Remove key from cache
func Remove(key string) {
if conn == nil {
if defaultCache == nil {
return
}
_ = conn.Delete(key)
_ = defaultCache.Delete(key)
}

2
modules/cache/cache_redis.go vendored
View File

@ -11,7 +11,7 @@ import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/nosql"
"gitea.com/go-chi/cache"
"gitea.com/go-chi/cache" //nolint:depguard
"github.com/redis/go-redis/v9"
)

40
modules/cache/cache_test.go vendored
View File

@ -14,7 +14,7 @@ import (
)
func createTestCache() {
conn, _ = newCache(setting.Cache{
defaultCache, _ = NewStringCache(setting.Cache{
Adapter: "memory",
TTL: time.Minute,
})
@ -25,7 +25,7 @@ func TestNewContext(t *testing.T) {
assert.NoError(t, Init())
setting.CacheService.Cache = setting.Cache{Adapter: "redis", Conn: "some random string"}
con, err := newCache(setting.Cache{
con, err := NewStringCache(setting.Cache{
Adapter: "rand",
Conn: "false conf",
Interval: 100,
@ -76,42 +76,6 @@ func TestGetString(t *testing.T) {
Remove("key")
}
func TestGetInt(t *testing.T) {
createTestCache()
data, err := GetInt("key", func() (int, error) {
return 0, fmt.Errorf("some error")
})
assert.Error(t, err)
assert.Equal(t, 0, data)
data, err = GetInt("key", func() (int, error) {
return 0, nil
})
assert.NoError(t, err)
assert.Equal(t, 0, data)
data, err = GetInt("key", func() (int, error) {
return 100, nil
})
assert.NoError(t, err)
assert.Equal(t, 0, data)
Remove("key")
data, err = GetInt("key", func() (int, error) {
return 100, nil
})
assert.NoError(t, err)
assert.Equal(t, 100, data)
data, err = GetInt("key", func() (int, error) {
return 0, fmt.Errorf("some error")
})
assert.NoError(t, err)
assert.Equal(t, 100, data)
Remove("key")
}
func TestGetInt64(t *testing.T) {
createTestCache()

2
modules/cache/cache_twoqueue.go vendored
View File

@ -10,7 +10,7 @@ import (
"code.gitea.io/gitea/modules/json"
mc "gitea.com/go-chi/cache"
mc "gitea.com/go-chi/cache" //nolint:depguard
lru "github.com/hashicorp/golang-lru/v2"
)

120
modules/cache/string_cache.go vendored Normal file
View File

@ -0,0 +1,120 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"errors"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
chi_cache "gitea.com/go-chi/cache" //nolint:depguard
)
type GetJSONError struct {
err error
cachedError string // Golang error can't be stored in cache, only the string message could be stored
}
func (e *GetJSONError) ToError() error {
if e.err != nil {
return e.err
}
return errors.New("cached error: " + e.cachedError)
}
type StringCache interface {
Ping() error
Get(key string) (string, bool)
Put(key, value string, ttl int64) error
Delete(key string) error
IsExist(key string) bool
PutJSON(key string, v any, ttl int64) error
GetJSON(key string, ptr any) (exist bool, err *GetJSONError)
ChiCache() chi_cache.Cache
}
type stringCache struct {
chiCache chi_cache.Cache
}
func NewStringCache(cacheConfig setting.Cache) (StringCache, error) {
adapter := util.IfZero(cacheConfig.Adapter, "memory")
interval := util.IfZero(cacheConfig.Interval, 60)
cc, err := chi_cache.NewCacher(chi_cache.Options{
Adapter: adapter,
AdapterConfig: cacheConfig.Conn,
Interval: interval,
})
if err != nil {
return nil, err
}
return &stringCache{chiCache: cc}, nil
}
func (sc *stringCache) Ping() error {
return sc.chiCache.Ping()
}
func (sc *stringCache) Get(key string) (string, bool) {
v := sc.chiCache.Get(key)
if v == nil {
return "", false
}
s, ok := v.(string)
return s, ok
}
func (sc *stringCache) Put(key, value string, ttl int64) error {
return sc.chiCache.Put(key, value, ttl)
}
func (sc *stringCache) Delete(key string) error {
return sc.chiCache.Delete(key)
}
func (sc *stringCache) IsExist(key string) bool {
return sc.chiCache.IsExist(key)
}
const cachedErrorPrefix = "<CACHED-ERROR>:"
func (sc *stringCache) PutJSON(key string, v any, ttl int64) error {
var s string
switch v := v.(type) {
case error:
s = cachedErrorPrefix + v.Error()
default:
b, err := json.Marshal(v)
if err != nil {
return err
}
s = util.UnsafeBytesToString(b)
}
return sc.chiCache.Put(key, s, ttl)
}
func (sc *stringCache) GetJSON(key string, ptr any) (exist bool, getErr *GetJSONError) {
s, ok := sc.Get(key)
if !ok || s == "" {
return false, nil
}
s, isCachedError := strings.CutPrefix(s, cachedErrorPrefix)
if isCachedError {
return true, &GetJSONError{cachedError: s}
}
if err := json.Unmarshal(util.UnsafeStringToBytes(s), ptr); err != nil {
return false, &GetJSONError{err: err}
}
return true, nil
}
func (sc *stringCache) ChiCache() chi_cache.Cache {
return sc.chiCache
}

15
modules/git/last_commit_cache.go
View File

@ -7,18 +7,11 @@ import (
"crypto/sha256"
"fmt"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
// Cache represents a caching interface
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val any, timeout int64) error
// Get gets cached value by given key.
Get(key string) any
}
func getCacheKey(repoPath, commitID, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
@ -30,11 +23,11 @@ type LastCommitCache struct {
ttl func() int64
repo *Repository
commitCache map[string]*Commit
cache Cache
cache cache.StringCache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache Cache) *LastCommitCache {
func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache cache.StringCache) *LastCommitCache {
if cache == nil {
return nil
}
@ -65,7 +58,7 @@ func (c *LastCommitCache) Get(ref, entryPath string) (*Commit, error) {
return nil, nil
}
commitID, ok := c.cache.Get(getCacheKey(c.repoPath, ref, entryPath)).(string)
commitID, ok := c.cache.Get(getCacheKey(c.repoPath, ref, entryPath))
if !ok || commitID == "" {
return nil, nil
}

6
routers/api/v1/misc/nodeinfo.go
View File

@ -29,9 +29,7 @@ func NodeInfo(ctx *context.APIContext) {
nodeInfoUsage := structs.NodeInfoUsage{}
if setting.Federation.ShareUserStatistics {
var cached bool
nodeInfoUsage, cached = ctx.Cache.Get(cacheKeyNodeInfoUsage).(structs.NodeInfoUsage)
cached, _ := ctx.Cache.GetJSON(cacheKeyNodeInfoUsage, &nodeInfoUsage)
if !cached {
usersTotal := int(user_model.CountUsers(ctx, nil))
now := time.Now()
@ -53,7 +51,7 @@ func NodeInfo(ctx *context.APIContext) {
LocalComments: int(allComments),
}
if err := ctx.Cache.Put(cacheKeyNodeInfoUsage, nodeInfoUsage, 180); err != nil {
if err := ctx.Cache.PutJSON(cacheKeyNodeInfoUsage, nodeInfoUsage, 180); err != nil {
ctx.InternalServerError(err)
return
}

8
services/context/api.go
View File

@ -13,7 +13,7 @@ import (
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
mc "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/httpcache"
@ -21,15 +21,13 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/web"
web_types "code.gitea.io/gitea/modules/web/types"
"gitea.com/go-chi/cache"
)
// APIContext is a specific context for API service
type APIContext struct {
*Base
Cache cache.Cache
Cache cache.StringCache
Doer *user_model.User // current signed-in user
IsSigned bool
@ -217,7 +215,7 @@ func APIContexter() func(http.Handler) http.Handler {
base, baseCleanUp := NewBaseContext(w, req)
ctx := &APIContext{
Base: base,
Cache: mc.GetCache(),
Cache: cache.GetCache(),
Repo: &Repository{PullRequest: &PullRequest{}},
Org: &APIOrganization{},
}

2
services/context/captcha.go
View File

@ -30,7 +30,7 @@ func GetImageCaptcha() *captcha.Captcha {
cpt = captcha.NewCaptcha(captcha.Options{
SubURL: setting.AppSubURL,
})
cpt.Store = cache.GetCache()
cpt.Store = cache.GetCache().ChiCache()
})
return cpt
}

7
services/context/context.go
View File

@ -17,7 +17,7 @@ import (
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
mc "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/setting"
@ -27,7 +27,6 @@ import (
"code.gitea.io/gitea/modules/web/middleware"
web_types "code.gitea.io/gitea/modules/web/types"
"gitea.com/go-chi/cache"
"gitea.com/go-chi/session"
)
@ -46,7 +45,7 @@ type Context struct {
Render Render
PageData map[string]any // data used by JavaScript modules in one page, it's `window.config.pageData`
Cache cache.Cache
Cache cache.StringCache
Csrf CSRFProtector
Flash *middleware.Flash
Session session.Store
@ -111,7 +110,7 @@ func NewWebContext(base *Base, render Render, session session.Store) *Context {
Render: render,
Session: session,
Cache: mc.GetCache(),
Cache: cache.GetCache(),
Link: setting.AppSubURL + strings.TrimSuffix(base.Req.URL.EscapedPath(), "/"),
Repo: &Repository{PullRequest: &PullRequest{}},
Org: &Organization{},

11
services/repository/branch.go
View File

@ -26,6 +26,7 @@ import (
"code.gitea.io/gitea/modules/queue"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
webhook_module "code.gitea.io/gitea/modules/webhook"
notify_service "code.gitea.io/gitea/services/notify"
files_service "code.gitea.io/gitea/services/repository/files"
@ -119,17 +120,15 @@ func getDivergenceCacheKey(repoID int64, branchName string) string {
// getDivergenceFromCache gets the divergence from cache
func getDivergenceFromCache(repoID int64, branchName string) (*git.DivergeObject, bool) {
data := cache.GetCache().Get(getDivergenceCacheKey(repoID, branchName))
data, ok := cache.GetCache().Get(getDivergenceCacheKey(repoID, branchName))
res := git.DivergeObject{
Ahead: -1,
Behind: -1,
}
s, ok := data.([]byte)
if !ok || len(s) == 0 {
if !ok || data == "" {
return &res, false
}
if err := json.Unmarshal(s, &res); err != nil {
if err := json.Unmarshal(util.UnsafeStringToBytes(data), &res); err != nil {
log.Error("json.UnMarshal failed: %v", err)
return &res, false
}
@ -141,7 +140,7 @@ func putDivergenceFromCache(repoID int64, branchName string, divergence *git.Div
if err != nil {
return err
}
return cache.GetCache().Put(getDivergenceCacheKey(repoID, branchName), bs, 30*24*60*60)
return cache.GetCache().Put(getDivergenceCacheKey(repoID, branchName), util.UnsafeBytesToString(bs), 30*24*60*60)
}
func DelDivergenceFromCache(repoID int64, branchName string) error {

2
services/repository/commitstatus/commitstatus.go
View File

@ -34,7 +34,7 @@ type commitStatusCacheValue struct {
func getCommitStatusCache(repoID int64, branchName string) *commitStatusCacheValue {
c := cache.GetCache()
statusStr, ok := c.Get(getCacheKey(repoID, branchName)).(string)
statusStr, ok := c.Get(getCacheKey(repoID, branchName))
if ok && statusStr != "" {
var cv commitStatusCacheValue
err := json.Unmarshal([]byte(statusStr), &cv)

32
services/repository/contributors_graph.go
View File

@ -17,13 +17,12 @@ import (
"code.gitea.io/gitea/models/avatars"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
api "code.gitea.io/gitea/modules/structs"
"gitea.com/go-chi/cache"
)
const (
@ -79,13 +78,13 @@ func findLastSundayBeforeDate(dateStr string) (string, error) {
}
// GetContributorStats returns contributors stats for git commits for given revision or default branch
func GetContributorStats(ctx context.Context, cache cache.Cache, repo *repo_model.Repository, revision string) (map[string]*ContributorData, error) {
func GetContributorStats(ctx context.Context, cache cache.StringCache, repo *repo_model.Repository, revision string) (map[string]*ContributorData, error) {
// as GetContributorStats is resource intensive we cache the result
cacheKey := fmt.Sprintf(contributorStatsCacheKey, repo.FullName(), revision)
if !cache.IsExist(cacheKey) {
genReady := make(chan struct{})
// dont start multible async generations
// dont start multiple async generations
_, run := generateLock.Load(cacheKey)
if run {
return nil, ErrAwaitGeneration
@ -104,15 +103,11 @@ func GetContributorStats(ctx context.Context, cache cache.Cache, repo *repo_mode
}
}
// TODO: renew timeout of cache cache.UpdateTimeout(cacheKey, contributorStatsCacheTimeout)
switch v := cache.Get(cacheKey).(type) {
case error:
return nil, v
case map[string]*ContributorData:
return v, nil
default:
return nil, fmt.Errorf("unexpected type in cache detected")
var res map[string]*ContributorData
if _, cacheErr := cache.GetJSON(cacheKey, &res); cacheErr != nil {
return nil, fmt.Errorf("cached error: %w", cacheErr.ToError())
}
return res, nil
}
// getExtendedCommitStats return the list of *ExtendedCommitStats for the given revision
@ -205,13 +200,12 @@ func getExtendedCommitStats(repo *git.Repository, revision string /*, limit int
return extendedCommitStats, nil
}
func generateContributorStats(genDone chan struct{}, cache cache.Cache, cacheKey string, repo *repo_model.Repository, revision string) {
func generateContributorStats(genDone chan struct{}, cache cache.StringCache, cacheKey string, repo *repo_model.Repository, revision string) {
ctx := graceful.GetManager().HammerContext()
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
if err != nil {
err := fmt.Errorf("OpenRepository: %w", err)
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
_ = cache.PutJSON(cacheKey, fmt.Errorf("OpenRepository: %w", err), contributorStatsCacheTimeout)
return
}
defer closer.Close()
@ -221,13 +215,11 @@ func generateContributorStats(genDone chan struct{}, cache cache.Cache, cacheKey
}
extendedCommitStats, err := getExtendedCommitStats(gitRepo, revision)
if err != nil {
err := fmt.Errorf("ExtendedCommitStats: %w", err)
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
_ = cache.PutJSON(cacheKey, fmt.Errorf("ExtendedCommitStats: %w", err), contributorStatsCacheTimeout)
return
}
if len(extendedCommitStats) == 0 {
err := fmt.Errorf("no commit stats returned for revision '%s'", revision)
_ = cache.Put(cacheKey, err, contributorStatsCacheTimeout)
_ = cache.PutJSON(cacheKey, fmt.Errorf("no commit stats returned for revision '%s'", revision), contributorStatsCacheTimeout)
return
}
@ -309,7 +301,7 @@ func generateContributorStats(genDone chan struct{}, cache cache.Cache, cacheKey
total.TotalCommits++
}
_ = cache.Put(cacheKey, contributorsCommitStats, contributorStatsCacheTimeout)
_ = cache.PutJSON(cacheKey, contributorsCommitStats, contributorStatsCacheTimeout)
generateLock.Delete(cacheKey)
if genDone != nil {
genDone <- struct{}{}

20
services/repository/contributors_graph_test.go
View File

@ -10,9 +10,9 @@ import (
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/setting"
"gitea.com/go-chi/cache"
"github.com/stretchr/testify/assert"
)
@ -20,20 +20,18 @@ func TestRepository_ContributorsGraph(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2})
assert.NoError(t, repo.LoadOwner(db.DefaultContext))
mockCache, err := cache.NewCacher(cache.Options{
Adapter: "memory",
Interval: 24 * 60,
})
mockCache, err := cache.NewStringCache(setting.Cache{})
assert.NoError(t, err)
generateContributorStats(nil, mockCache, "key", repo, "404ref")
err, isErr := mockCache.Get("key").(error)
assert.True(t, isErr)
assert.ErrorAs(t, err, &git.ErrNotExist{})
var data map[string]*ContributorData
_, getErr := mockCache.GetJSON("key", &data)
assert.NotNil(t, getErr)
assert.ErrorContains(t, getErr.ToError(), "object does not exist")
generateContributorStats(nil, mockCache, "key2", repo, "master")
data, isData := mockCache.Get("key2").(map[string]*ContributorData)
assert.True(t, isData)
exist, _ := mockCache.GetJSON("key2", &data)
assert.True(t, exist)
var keys []string
for k := range data {
keys = append(keys, k)