linden123/githaven-fork
1
1
Fork 0
forked from Shiloh/githaven
Code Pull Requests Activity

Integrate alpine noarch packages into other architectures index (#29137)

Browse Source 
Fixes #26691
Revert #24972

The alpine package manager expects `noarch` packages in the index of
other architectures too.

---------

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
KN4CK3R 2024-02-25 16:10:55 +01:00 committed by GitHub
parent 756b952c52
commit 1c6858543c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 190 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/packages/alpine/metadata.go
View File

@ -34,6 +34,8 @@ const (
RepositoryPackage = "_alpine"
RepositoryVersion = "_repository"
NoArch = "noarch"
)
// https://wiki.alpinelinux.org/wiki/Apk_spec

29
routers/api/packages/alpine/alpine.go
View File

@ -72,7 +72,7 @@ func GetRepositoryFile(ctx *context.Context) {
ctx,
pv,
&packages_service.PackageFileInfo{
Filename: alpine_service.IndexFilename,
Filename: alpine_service.IndexArchiveFilename,
CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")),
},
)
@ -182,21 +182,40 @@ func UploadPackageFile(ctx *context.Context) {
}
func DownloadPackageFile(ctx *context.Context) {
pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
branch := ctx.Params("branch")
repository := ctx.Params("repository")
architecture := ctx.Params("architecture")
opts := &packages_model.PackageFileSearchOptions{
OwnerID: ctx.Package.Owner.ID,
PackageType: packages_model.TypeAlpine,
Query: ctx.Params("filename"),
CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")),
})
CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture),
}
pfs, _, err := packages_model.SearchFiles(ctx, opts)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
if len(pfs) != 1 {
if len(pfs) == 0 {
// Try again with architecture 'noarch'
if architecture == alpine_module.NoArch {
apiError(ctx, http.StatusNotFound, nil)
return
}
opts.CompositeKey = fmt.Sprintf("%s|%s|%s", branch, repository, alpine_module.NoArch)
if pfs, _, err = packages_model.SearchFiles(ctx, opts); err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
if len(pfs) == 0 {
apiError(ctx, http.StatusNotFound, nil)
return
}
}
s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0])
if err != nil {
if errors.Is(err, util.ErrNotExist) {

56
services/packages/alpine/repository.go
View File

@ -23,6 +23,7 @@ import (
packages_model "code.gitea.io/gitea/models/packages"
alpine_model "code.gitea.io/gitea/models/packages/alpine"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/json"
packages_module "code.gitea.io/gitea/modules/packages"
alpine_module "code.gitea.io/gitea/modules/packages/alpine"
@ -30,7 +31,10 @@ import (
packages_service "code.gitea.io/gitea/services/packages"
)
const IndexFilename = "APKINDEX.tar.gz"
const (
IndexFilename = "APKINDEX"
IndexArchiveFilename = IndexFilename + ".tar.gz"
)
// GetOrCreateRepositoryVersion gets or creates the internal repository package
// The Alpine registry needs multiple index files which are stored in this package.
@ -120,7 +124,22 @@ func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, branch, re
return err
}
return buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture)
architectures := container.SetOf(architecture)
if architecture == alpine_module.NoArch {
// Update all other architectures too when updating the noarch index
additionalArchitectures, err := alpine_model.GetArchitectures(ctx, ownerID, repository)
if err != nil {
return err
}
architectures.AddMultiple(additionalArchitectures...)
}
for architecture := range architectures {
if err := buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture); err != nil {
return err
}
}
return nil
}
type packageData struct {
@ -133,8 +152,7 @@ type packageData struct {
type packageCache = map[*packages_model.PackageFile]*packageData
// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format
func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error {
func searchPackageFiles(ctx context.Context, ownerID int64, branch, repository, architecture string) ([]*packages_model.PackageFile, error) {
pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
OwnerID: ownerID,
PackageType: packages_model.TypeAlpine,
@ -145,13 +163,30 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package
alpine_module.PropertyArchitecture: architecture,
},
})
if err != nil {
return nil, err
}
return pfs, nil
}
// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format
func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error {
pfs, err := searchPackageFiles(ctx, ownerID, branch, repository, architecture)
if err != nil {
return err
}
if architecture != alpine_module.NoArch {
// Add all noarch packages too
noarchFiles, err := searchPackageFiles(ctx, ownerID, branch, repository, alpine_module.NoArch)
if err != nil {
return err
}
pfs = append(pfs, noarchFiles...)
}
// Delete the package indices if there are no packages
if len(pfs) == 0 {
pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture))
pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexArchiveFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture))
if err != nil && !errors.Is(err, util.ErrNotExist) {
return err
} else if pf == nil {
@ -206,7 +241,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package
fmt.Fprintf(&buf, "C:%s\n", pd.FileMetadata.Checksum)
fmt.Fprintf(&buf, "P:%s\n", pd.Package.Name)
fmt.Fprintf(&buf, "V:%s\n", pd.Version.Version)
fmt.Fprintf(&buf, "A:%s\n", pd.FileMetadata.Architecture)
fmt.Fprintf(&buf, "A:%s\n", architecture)
if pd.VersionMetadata.Description != "" {
fmt.Fprintf(&buf, "T:%s\n", pd.VersionMetadata.Description)
}
@ -244,7 +279,7 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package
h := sha1.New()
if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), "APKINDEX", buf.Bytes(), true); err != nil {
if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), IndexFilename, buf.Bytes(), true); err != nil {
return err
}
@ -299,13 +334,18 @@ func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *package
repoVersion,
&packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{
Filename: IndexFilename,
Filename: IndexArchiveFilename,
CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture),
},
Creator: user_model.NewGhostUser(),
Data: signedIndexContent,
IsLead: false,
OverwriteExisting: true,
Properties: map[string]string{
alpine_module.PropertyBranch: branch,
alpine_module.PropertyRepository: repository,
alpine_module.PropertyArchitecture: architecture,
},
},
)
return err

133
tests/integration/api_packages_alpine_test.go
View File

@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
alpine_module "code.gitea.io/gitea/modules/packages/alpine"
alpine_service "code.gitea.io/gitea/services/packages/alpine"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
@ -59,7 +60,34 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA`
content, err := base64.StdEncoding.DecodeString(base64AlpinePackageContent)
assert.NoError(t, err)
branches := []string{"v3.16", "v3.17", "v3.18"}
base64AlpinePackageNoArchContent := `H4sIAAAAAAACA9ML9nT30wsKdtQrLU4t0jUzTUo1NDVP0ysqTtQrKE1ioAYwAAIzExMwDQTotCGI
bWhiampuYmRiaGrMYGBoZGZkxKBgwEAHUFpcklikoMAwQkHLB7eoE40P9n5jvx32t7Dy9rq7x19k
66cJPV38t/h+vWe2jdXy+/PzPT0YTF5z39i4cPFptcLa1C1lD0z/XvrNp6In/7nP4PPCF2pZu8uV
z74QXLxpY1XWJuVFysqVf+PdizccFbD6ZL/QPGXd1Ri1fec2XBNuYfK/rFa6wF/h3dK/W12f8mxP
04iP3aCy+vPx7h9S+5M1LLkWr5M/4ezGt3bDW/FjBp/S9hiKP72s/XrJ0vWtO0zr5wa+D/X8XluW
d7BLP7XS3YUhd8WbPPF/NW3691ONJbXsRb69O7BIMZC96uTri+utC/fbie5J+n7zhCxD4Aep/qet
QnlCZyN8MhNdVNlNl7965R1nExrrGvfI/YQZFx8Dg+d9122hZsYd/24WL/L69OWrDAN/y//nS7im
XEive3v7QeTe433TPj/X71+9yHiV6+E9k++3TL8V0Xoq9panhNt23fLgau/pTOvmKx6bV/pS26+Y
5UP4viyuklYeu4/BZl6rLINe1L/uWuUXcH5z7pa2b9+/rp/v/8dFgc1PL3bO3/iVcrI//J/LMU2X
Nzu1IaMmWXnGp7CmyQIR39d0Nai9/+tdPbfjvmsNH88Tu7uVrvNuJE0wjxfePXGv/KHNXD+mnG0t
yTPu+Na0b5WR9O4t0yMd9T5k6ui7hOyU/jL/4dOn6neLwhdrZIZfcl1ectnGvUTurWDo1vY5Gw9k
PTQLVgcA61F+7gAEAAAfiwgAAAAAAAID7VVNa9wwEPXZv2Ig53hHlizbCzkVkobQJtDkB4wl2SvW
lhdbTpP++oyXQGEPLYU2paTvIs3X05PQSNnmjp4+OrJumjfZ3c3V9efL2+T3AhlaqePIOB0Rc50I
VRSlypUoZIJCKJQJPCVvgGWONLGU5H1CCDDRD+4CU57S6zT5j3eCP9Tyv9T/GsuT/scyLxPAt+z/
aRzjj/J+Fv9HcQZXLriJorPQPAM1i+8tyEzkGZ5PmJ7BMvvQQUt7tx4BPPJH4ccAIpN5Jjj+hSJc
ugZAghDbArco4eH+A+SYq/Sw7wINDi6g89HReRhpMrvVzTzsFZlaV2Hbutmw4zVhmXo2djEe5u1m
c6zNzDikR3mW1a61JepaC0SZHsjsqTsyPoR9GL+GdPbf1iSFtU5Xyu/c4+Q7H04lMfvgI3vT3hsX
5rX40/U9b5CWOA78Mhrq+2ewLjrDp7VNWQbtaF6ZXVWZIhdV09RWOIvU6BqNboSxLSEpkrpQq80x
W1Nla6NavuqtrJQ0sv17D+4L2oD1lwAIAAAfiwgAAAAAAAID7dM/SgNBFAbw6cSAnYXlXsDNm50/
u1METBeIkEBMK87uzKKEJbB/IN7CxhN4AI/gNcRD6BWciI0WSiBGxO/XvA9mile8L+5P7WrkrfN1
049dV1XXbNso0FK+zeDzJC4SxqVSqUwkV4IR51KkLFqxHeia1tZhFfY/cR4V7VXlB9QL0b5HnUXD
6fj4bDI5ncXFpS8WTVfFs9GQD5wVxgrvlde5zMmJRKm89KVRmnhmyJYuo5RMj8Ef8EOV36j/6/yx
/5qnxKJ1J8MZJifskD2Zu+fzxfggmT+83F4c3dw/7u1vtf/1ctl+9e+7dwAAAAAAAAAAAAAAAAAA
AACAX/AKARNTyAAoAAA=`
noarchContent, err := base64.StdEncoding.DecodeString(base64AlpinePackageNoArchContent)
assert.NoError(t, err)
branches := []string{"v3.16", "v3.17"}
repositories := []string{"main", "testing"}
rootURL := fmt.Sprintf("/api/packages/%s/alpine", user.Name)
@ -139,19 +167,13 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA`
})
})
t.Run("Index", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
url := fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository)
req := NewRequest(t, "GET", url)
resp := MakeRequest(t, req, http.StatusOK)
assert.Condition(t, func() bool {
br := bufio.NewReader(resp.Body)
readIndexContent := func(r io.Reader) (string, error) {
br := bufio.NewReader(r)
gzr, err := gzip.NewReader(br)
assert.NoError(t, err)
if err != nil {
return "", err
}
for {
gzr.Multistream(false)
@ -162,28 +184,17 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA`
if err == io.EOF {
break
}
assert.NoError(t, err)
if err != nil {
return "", err
}
if hd.Name == "APKINDEX" {
if hd.Name == alpine_service.IndexFilename {
buf, err := io.ReadAll(tr)
assert.NoError(t, err)
if err != nil {
return "", err
}
s := string(buf)
assert.Contains(t, s, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n")
assert.Contains(t, s, "P:"+packageName+"\n")
assert.Contains(t, s, "V:"+packageVersion+"\n")
assert.Contains(t, s, "A:x86_64\n")
assert.Contains(t, s, "T:Gitea Test Package\n")
assert.Contains(t, s, "U:https://gitea.io/\n")
assert.Contains(t, s, "L:MIT\n")
assert.Contains(t, s, "S:1353\n")
assert.Contains(t, s, "I:4096\n")
assert.Contains(t, s, "o:gitea-test\n")
assert.Contains(t, s, "m:KN4CK3R <kn4ck3r@gitea.io>\n")
assert.Contains(t, s, "t:1679498030\n")
return true
return string(buf), nil
}
}
@ -191,11 +202,36 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA`
if err == io.EOF {
break
}
assert.NoError(t, err)
if err != nil {
return "", err
}
}
return false
})
return "", io.EOF
}
t.Run("Index", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository))
resp := MakeRequest(t, req, http.StatusOK)
content, err := readIndexContent(resp.Body)
assert.NoError(t, err)
assert.Contains(t, content, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n")
assert.Contains(t, content, "P:"+packageName+"\n")
assert.Contains(t, content, "V:"+packageVersion+"\n")
assert.Contains(t, content, "A:x86_64\n")
assert.NotContains(t, content, "A:noarch\n")
assert.Contains(t, content, "T:Gitea Test Package\n")
assert.Contains(t, content, "U:https://gitea.io/\n")
assert.Contains(t, content, "L:MIT\n")
assert.Contains(t, content, "S:1353\n")
assert.Contains(t, content, "I:4096\n")
assert.Contains(t, content, "o:gitea-test\n")
assert.Contains(t, content, "m:KN4CK3R <kn4ck3r@gitea.io>\n")
assert.Contains(t, content, "t:1679498030\n")
})
t.Run("Download", func(t *testing.T) {
@ -204,6 +240,35 @@ Djfa/2q5bH4699v++uMAAAAAAAAAAAAAAAAAAAAAAHbgA/eXQh8AKAAA`
req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/%s-%s.apk", rootURL, branch, repository, packageName, packageVersion))
MakeRequest(t, req, http.StatusOK)
})
t.Run("NoArch", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/%s/%s", rootURL, branch, repository), bytes.NewReader(noarchContent)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/x86_64/APKINDEX.tar.gz", rootURL, branch, repository))
resp := MakeRequest(t, req, http.StatusOK)
content, err := readIndexContent(resp.Body)
assert.NoError(t, err)
assert.Contains(t, content, "C:Q1/se1PjO94hYXbfpNR1/61hVORIc=\n")
assert.Contains(t, content, "A:x86_64\n")
assert.Contains(t, content, "C:Q1kbH5WoIPFccQYyATanaKXd2cJcc=\n")
assert.NotContains(t, content, "A:noarch\n")
// noarch package should be available with every architecture requested
for _, arch := range []string{alpine_module.NoArch, "x86_64", "my_arch"} {
req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/%s/gitea-noarch-1.4-r0.apk", rootURL, branch, repository, arch))
MakeRequest(t, req, http.StatusOK)
}
req = NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/noarch/gitea-noarch-1.4-r0.apk", rootURL, branch, repository)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
})
})
}
}