From 4197e2810081025a6614624e7b1731af91c8db72 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 24 Feb 2024 19:46:49 +0100 Subject: [PATCH] Refactor git attributes (#29356) --- modules/git/attribute.go | 35 ++++++++++++++++++ modules/git/repo_attribute.go | 36 ++++++------------- modules/git/repo_attribute_test.go | 10 +++--- modules/git/repo_language_stats.go | 19 ++++++++++ modules/git/repo_language_stats_gogit.go | 20 +++-------- modules/git/repo_language_stats_nogogit.go | 20 +++-------- routers/web/repo/view.go | 7 ++-- services/gitdiff/gitdiff.go | 42 +++++++++------------- services/repository/files/content.go | 12 ++----- 9 files changed, 101 insertions(+), 100 deletions(-) create mode 100644 modules/git/attribute.go diff --git a/modules/git/attribute.go b/modules/git/attribute.go new file mode 100644 index 000000000..4dfa51036 --- /dev/null +++ b/modules/git/attribute.go @@ -0,0 +1,35 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "code.gitea.io/gitea/modules/optional" +) + +const ( + AttributeLinguistVendored = "linguist-vendored" + AttributeLinguistGenerated = "linguist-generated" + AttributeLinguistDocumentation = "linguist-documentation" + AttributeLinguistDetectable = "linguist-detectable" + AttributeLinguistLanguage = "linguist-language" + AttributeGitlabLanguage = "gitlab-language" +) + +// true if "set"/"true", false if "unset"/"false", none otherwise +func AttributeToBool(attr map[string]string, name string) optional.Option[bool] { + switch attr[name] { + case "set", "true": + return optional.Some(true) + case "unset", "false": + return optional.Some(false) + } + return optional.None[bool]() +} + +func AttributeToString(attr map[string]string, name string) optional.Option[string] { + if value, has := attr[name]; has && value != "unspecified" { + return optional.Some(value) + } + return optional.None[string]() +} diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go index 44f13ddc2..84f85d1b1 100644 --- a/modules/git/repo_attribute.go +++ b/modules/git/repo_attribute.go @@ -11,7 +11,6 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/optional" ) // CheckAttributeOpts represents the possible options to CheckAttribute @@ -292,10 +291,17 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe } checker := &CheckAttributeReader{ - Attributes: []string{"linguist-vendored", "linguist-generated", "linguist-language", "gitlab-language", "linguist-documentation", "linguist-detectable"}, - Repo: repo, - IndexFile: indexFilename, - WorkTree: worktree, + Attributes: []string{ + AttributeLinguistVendored, + AttributeLinguistGenerated, + AttributeLinguistDocumentation, + AttributeLinguistDetectable, + AttributeLinguistLanguage, + AttributeGitlabLanguage, + }, + Repo: repo, + IndexFile: indexFilename, + WorkTree: worktree, } ctx, cancel := context.WithCancel(repo.Ctx) if err := checker.Init(ctx); err != nil { @@ -317,23 +323,3 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe return checker, deferable } - -// true if "set"/"true", false if "unset"/"false", none otherwise -func attributeToBool(attr map[string]string, name string) optional.Option[bool] { - if value, has := attr[name]; has && value != "unspecified" { - switch value { - case "set", "true": - return optional.Some(true) - case "unset", "false": - return optional.Some(false) - } - } - return optional.None[bool]() -} - -func attributeToString(attr map[string]string, name string) optional.Option[string] { - if value, has := attr[name]; has && value != "unspecified" { - return optional.Some(value) - } - return optional.None[string]() -} diff --git a/modules/git/repo_attribute_test.go b/modules/git/repo_attribute_test.go index ed16dccbe..0fcd94b4c 100644 --- a/modules/git/repo_attribute_test.go +++ b/modules/git/repo_attribute_test.go @@ -24,7 +24,7 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { select { case attr := <-wr.ReadAttribute(): assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, "linguist-vendored", attr.Attribute) + assert.Equal(t, AttributeLinguistVendored, attr.Attribute) assert.Equal(t, "unspecified", attr.Value) case <-time.After(100 * time.Millisecond): assert.FailNow(t, "took too long to read an attribute from the list") @@ -38,7 +38,7 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { select { case attr := <-wr.ReadAttribute(): assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, "linguist-vendored", attr.Attribute) + assert.Equal(t, AttributeLinguistVendored, attr.Attribute) assert.Equal(t, "unspecified", attr.Value) case <-time.After(100 * time.Millisecond): assert.FailNow(t, "took too long to read an attribute from the list") @@ -77,21 +77,21 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, attributeTriple{ Filename: "shouldbe.vendor", - Attribute: "linguist-vendored", + Attribute: AttributeLinguistVendored, Value: "set", }, attr) attr = <-wr.ReadAttribute() assert.NoError(t, err) assert.EqualValues(t, attributeTriple{ Filename: "shouldbe.vendor", - Attribute: "linguist-generated", + Attribute: AttributeLinguistGenerated, Value: "unspecified", }, attr) attr = <-wr.ReadAttribute() assert.NoError(t, err) assert.EqualValues(t, attributeTriple{ Filename: "shouldbe.vendor", - Attribute: "linguist-language", + Attribute: AttributeLinguistLanguage, Value: "unspecified", }, attr) } diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go index c40d6937b..8551ea9d2 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/repo_language_stats.go @@ -6,6 +6,8 @@ package git import ( "strings" "unicode" + + "code.gitea.io/gitea/modules/optional" ) const ( @@ -46,3 +48,20 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 { } return res } + +func TryReadLanguageAttribute(attrs map[string]string) optional.Option[string] { + language := AttributeToString(attrs, AttributeLinguistLanguage) + if language.Value() == "" { + language = AttributeToString(attrs, AttributeGitlabLanguage) + if language.Has() { + raw := language.Value() + // gitlab-language may have additional parameters after the language + // ignore them and just use the main language + // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type + if idx := strings.IndexByte(raw, '?'); idx >= 0 { + language = optional.Some(raw[:idx]) + } + } + } + return language +} diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/repo_language_stats_gogit.go index 99c7a894d..a34c03c78 100644 --- a/modules/git/repo_language_stats_gogit.go +++ b/modules/git/repo_language_stats_gogit.go @@ -8,7 +8,6 @@ package git import ( "bytes" "io" - "strings" "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/optional" @@ -66,36 +65,27 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if checker != nil { attrs, err := checker.CheckPath(f.Name) if err == nil { - isVendored = attributeToBool(attrs, "linguist-vendored") + isVendored = AttributeToBool(attrs, AttributeLinguistVendored) if isVendored.ValueOrDefault(false) { return nil } - isGenerated = attributeToBool(attrs, "linguist-generated") + isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) if isGenerated.ValueOrDefault(false) { return nil } - isDocumentation = attributeToBool(attrs, "linguist-documentation") + isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) if isDocumentation.ValueOrDefault(false) { return nil } - isDetectable = attributeToBool(attrs, "linguist-detectable") + isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) if !isDetectable.ValueOrDefault(true) { return nil } - hasLanguage := attributeToString(attrs, "linguist-language") - if hasLanguage.Value() == "" { - hasLanguage = attributeToString(attrs, "gitlab-language") - if hasLanguage.Has() { - language := hasLanguage.Value() - if idx := strings.IndexByte(language, '?'); idx >= 0 { - hasLanguage = optional.Some(language[:idx]) - } - } - } + hasLanguage := TryReadLanguageAttribute(attrs) if hasLanguage.Value() != "" { language := hasLanguage.Value() diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index 16669924d..318fc091c 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -8,7 +8,6 @@ package git import ( "bytes" "io" - "strings" "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/log" @@ -97,36 +96,27 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if checker != nil { attrs, err := checker.CheckPath(f.Name()) if err == nil { - isVendored = attributeToBool(attrs, "linguist-vendored") + isVendored = AttributeToBool(attrs, AttributeLinguistVendored) if isVendored.ValueOrDefault(false) { continue } - isGenerated = attributeToBool(attrs, "linguist-generated") + isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) if isGenerated.ValueOrDefault(false) { continue } - isDocumentation = attributeToBool(attrs, "linguist-documentation") + isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) if isDocumentation.ValueOrDefault(false) { continue } - isDetectable = attributeToBool(attrs, "linguist-detectable") + isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) if !isDetectable.ValueOrDefault(true) { continue } - hasLanguage := attributeToString(attrs, "linguist-language") - if hasLanguage.Value() == "" { - hasLanguage = attributeToString(attrs, "gitlab-language") - if hasLanguage.Has() { - language := hasLanguage.Value() - if idx := strings.IndexByte(language, '?'); idx >= 0 { - hasLanguage = optional.Some(language[:idx]) - } - } - } + hasLanguage := TryReadLanguageAttribute(attrs) if hasLanguage.Value() != "" { language := hasLanguage.Value() diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 33a5941d3..48a35dd06 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -635,11 +635,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { defer deferable() attrs, err := checker.CheckPath(ctx.Repo.TreePath) if err == nil { - vendored, has := attrs["linguist-vendored"] - ctx.Data["IsVendored"] = has && (vendored == "set" || vendored == "true") - - generated, has := attrs["linguist-generated"] - ctx.Data["IsGenerated"] = has && (generated == "set" || generated == "true") + ctx.Data["IsVendored"] = git.AttributeToBool(attrs, git.AttributeLinguistVendored).Value() + ctx.Data["IsGenerated"] = git.AttributeToBool(attrs, git.AttributeLinguistGenerated).Value() } } } diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 0f6e2b6c1..740c74834 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" @@ -1181,41 +1182,30 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi for _, diffFile := range diff.Files { - gotVendor := false - gotGenerated := false + isVendored := optional.None[bool]() + isGenerated := optional.None[bool]() if checker != nil { attrs, err := checker.CheckPath(diffFile.Name) if err == nil { - if vendored, has := attrs["linguist-vendored"]; has { - if vendored == "set" || vendored == "true" { - diffFile.IsVendored = true - gotVendor = true - } else { - gotVendor = vendored == "false" - } - } - if generated, has := attrs["linguist-generated"]; has { - if generated == "set" || generated == "true" { - diffFile.IsGenerated = true - gotGenerated = true - } else { - gotGenerated = generated == "false" - } - } - if language, has := attrs["linguist-language"]; has && language != "unspecified" && language != "" { - diffFile.Language = language - } else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" { - diffFile.Language = language + isVendored = git.AttributeToBool(attrs, git.AttributeLinguistVendored) + isGenerated = git.AttributeToBool(attrs, git.AttributeLinguistGenerated) + + language := git.TryReadLanguageAttribute(attrs) + if language.Has() { + diffFile.Language = language.Value() } } } - if !gotVendor { - diffFile.IsVendored = analyze.IsVendor(diffFile.Name) + if !isVendored.Has() { + isVendored = optional.Some(analyze.IsVendor(diffFile.Name)) } - if !gotGenerated { - diffFile.IsGenerated = analyze.IsGenerated(diffFile.Name) + diffFile.IsVendored = isVendored.Value() + + if !isGenerated.Has() { + isGenerated = optional.Some(analyze.IsGenerated(diffFile.Name)) } + diffFile.IsGenerated = isGenerated.Value() tailSection := diffFile.GetTailSection(gitRepo, opts.BeforeCommitID, opts.AfterCommitID) if tailSection != nil { diff --git a/services/repository/files/content.go b/services/repository/files/content.go index f2a767768..9500b8f46 100644 --- a/services/repository/files/content.go +++ b/services/repository/files/content.go @@ -282,7 +282,7 @@ func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) ( filename2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{ CachedOnly: true, - Attributes: []string{"linguist-language", "gitlab-language"}, + Attributes: []string{git.AttributeLinguistLanguage, git.AttributeGitlabLanguage}, Filenames: []string{treePath}, IndexFile: indexFilename, WorkTree: worktree, @@ -291,13 +291,7 @@ func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) ( return "", err } - language := filename2attribute2info[treePath]["linguist-language"] - if language == "" || language == "unspecified" { - language = filename2attribute2info[treePath]["gitlab-language"] - } - if language == "unspecified" { - language = "" - } + language := git.TryReadLanguageAttribute(filename2attribute2info[treePath]) - return language, nil + return language.Value(), nil }