Shiloh/githaven
Shiloh/githaven
1
0
Fork 1
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

List all Debian package versions in Packages (#27786)

Browse Source 
Closes #27783

This PR lists all and not only the latest package versions in the
`Packages` index.
This commit is contained in:
KN4CK3R 2023-10-29 15:14:47 +01:00 committed by GitHub
parent 02dae3f84b
commit b36e2ca419
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 70 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
models/packages/container/search.go
View File

@ -207,6 +207,9 @@ func (opts *ImageTagsSearchOptions) configureOrderBy(e db.Engine) {
default:
e.Desc("package_version.created_unix")
}
// Sort by id for stable order with duplicates in the other field
e.Asc("package_version.id")
}
// SearchImageTags gets a sorted list of the tags of an image

45
models/packages/debian/search.go
View File

@ -21,8 +21,7 @@ type PackageSearchOptions struct {
Architecture string
}
// SearchLatestPackages gets the latest packages matching the search options
func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*packages.PackageFileDescriptor, error) {
func (opts *PackageSearchOptions) toCond() builder.Cond {
var cond builder.Cond = builder.Eq{
"package_file.is_lead": true,
"package.type": packages.TypeDebian,
@ -62,28 +61,40 @@ func SearchLatestPackages(ctx context.Context, opts *PackageSearchOptions) ([]*p
})
}
cond = cond.
And(builder.Expr("pv2.id IS NULL"))
return cond
}
joinCond := builder.
Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))").
And(builder.Eq{"pv2.is_internal": false})
// ExistPackages tests if there are packages matching the search options
func ExistPackages(ctx context.Context, opts *PackageSearchOptions) (bool, error) {
return db.GetEngine(ctx).
Table("package_file").
Join("INNER", "package_version", "package_version.id = package_file.version_id").
Join("INNER", "package", "package.id = package_version.package_id").
Where(opts.toCond()).
Exist(new(packages.PackageFile))
}
pfs := make([]*packages.PackageFile, 0, 10)
err := db.GetEngine(ctx).
// SearchPackages gets the packages matching the search options
func SearchPackages(ctx context.Context, opts *PackageSearchOptions, iter func(*packages.PackageFileDescriptor)) error {
return db.GetEngine(ctx).
Table("package_file").
Select("package_file.*").
Join("INNER", "package_version", "package_version.id = package_file.version_id").
Join("LEFT", "package_version pv2", joinCond).
Join("INNER", "package", "package.id = package_version.package_id").
Where(cond).
Desc("package_version.created_unix").
Find(&pfs)
if err != nil {
return nil, err
}
Where(opts.toCond()).
Asc("package.lower_name", "package_version.created_unix").
Iterate(new(packages.PackageFile), func(_ int, bean any) error {
pf := bean.(*packages.PackageFile)
return packages.GetPackageFileDescriptors(ctx, pfs)
pfd, err := packages.GetPackageFileDescriptor(ctx, pf)
if err != nil {
return err
}
iter(pfd)
return nil
})
}
// GetDistributions gets all available distributions

3
models/packages/package_version.go
View File

@ -278,6 +278,9 @@ func (opts *PackageSearchOptions) configureOrderBy(e db.Engine) {
default:
e.Desc("package_version.created_unix")
}
// Sort by id for stable order with duplicates in the other field
e.Asc("package_version.id")
}
// SearchVersions gets all versions of packages matching the search options

15
services/packages/debian/repository.go
View File

@ -165,18 +165,17 @@ func buildRepositoryFiles(ctx context.Context, ownerID int64, repoVersion *packa
// https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices
func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, distribution, component, architecture string) error {
pfds, err := debian_model.SearchLatestPackages(ctx, &debian_model.PackageSearchOptions{
opts := &debian_model.PackageSearchOptions{
OwnerID: ownerID,
Distribution: distribution,
Component: component,
Architecture: architecture,
})
if err != nil {
return err
}
// Delete the package indices if there are no packages
if len(pfds) == 0 {
if has, err := debian_model.ExistPackages(ctx, opts); err != nil {
return err
} else if !has {
key := fmt.Sprintf("%s|%s|%s", distribution, component, architecture)
for _, filename := range []string{"Packages", "Packages.gz", "Packages.xz"} {
pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, filename, key)
@ -211,7 +210,7 @@ func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packa
w := io.MultiWriter(packagesContent, gzw, xzw)
addSeparator := false
for _, pfd := range pfds {
if err := debian_model.SearchPackages(ctx, opts, func(pfd *packages_model.PackageFileDescriptor) {
if addSeparator {
fmt.Fprintln(w)
}
@ -225,6 +224,8 @@ func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packa
fmt.Fprintf(w, "SHA1: %s\n", pfd.Blob.HashSHA1)
fmt.Fprintf(w, "SHA256: %s\n", pfd.Blob.HashSHA256)
fmt.Fprintf(w, "SHA512: %s\n", pfd.Blob.HashSHA512)
}); err != nil {
return err
}
gzw.Close()
@ -238,7 +239,7 @@ func buildPackagesIndices(ctx context.Context, ownerID int64, repoVersion *packa
{"Packages.gz", packagesGzipContent},
{"Packages.xz", packagesXzContent},
} {
_, err = packages_service.AddFileToPackageVersionInternal(
_, err := packages_service.AddFileToPackageVersionInternal(
ctx,
repoVersion,
&packages_service.PackageFileCreationInfo{

46
tests/integration/api_packages_debian_test.go
View File

@ -31,6 +31,7 @@ func TestPackageDebian(t *testing.T) {
packageName := "gitea"
packageVersion := "1.0.3"
packageVersion2 := "1.0.4"
packageDescription := "Package Description"
createArchive := func(name, version, architecture string) io.Reader {
@ -80,11 +81,11 @@ func TestPackageDebian(t *testing.T) {
for _, component := range components {
for _, architecture := range architectures {
t.Run(fmt.Sprintf("[Component:%s,Architecture:%s]", component, architecture), func(t *testing.T) {
uploadURL := fmt.Sprintf("%s/pool/%s/%s/upload", rootURL, distribution, component)
t.Run("Upload", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
uploadURL := fmt.Sprintf("%s/pool/%s/%s/upload", rootURL, distribution, component)
req := NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader([]byte{}))
MakeRequest(t, req, http.StatusUnauthorized)
@ -100,18 +101,17 @@ func TestPackageDebian(t *testing.T) {
AddBasicAuthHeader(req, user.Name)
MakeRequest(t, req, http.StatusCreated)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeDebian)
pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeDebian, packageName, packageVersion)
assert.NoError(t, err)
assert.Len(t, pvs, 1)
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pv)
assert.NoError(t, err)
assert.Nil(t, pd.SemVer)
assert.IsType(t, &debian_module.Metadata{}, pd.Metadata)
assert.Equal(t, packageName, pd.Package.Name)
assert.Equal(t, packageVersion, pd.Version.Version)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID)
assert.NoError(t, err)
assert.NotEmpty(t, pfs)
assert.Condition(t, func() bool {
@ -162,17 +162,23 @@ func TestPackageDebian(t *testing.T) {
t.Run("Packages", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion2, architecture))
AddBasicAuthHeader(req, user.Name)
MakeRequest(t, req, http.StatusCreated)
url := fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture)
req := NewRequest(t, "GET", url)
req = NewRequest(t, "GET", url)
resp := MakeRequest(t, req, http.StatusOK)
body := resp.Body.String()
assert.Contains(t, body, "Package: "+packageName)
assert.Contains(t, body, "Version: "+packageVersion)
assert.Contains(t, body, "Architecture: "+architecture)
assert.Contains(t, body, fmt.Sprintf("Filename: pool/%s/%s/%s_%s_%s.deb", distribution, component, packageName, packageVersion, architecture))
assert.Contains(t, body, "Package: "+packageName+"\n")
assert.Contains(t, body, "Version: "+packageVersion+"\n")
assert.Contains(t, body, "Version: "+packageVersion2+"\n")
assert.Contains(t, body, "Architecture: "+architecture+"\n")
assert.Contains(t, body, fmt.Sprintf("Filename: pool/%s/%s/%s_%s_%s.deb\n", distribution, component, packageName, packageVersion, architecture))
assert.Contains(t, body, fmt.Sprintf("Filename: pool/%s/%s/%s_%s_%s.deb\n", distribution, component, packageName, packageVersion2, architecture))
req = NewRequest(t, "GET", url+".gz")
MakeRequest(t, req, http.StatusOK)
@ -198,14 +204,14 @@ func TestPackageDebian(t *testing.T) {
body := resp.Body.String()
assert.Contains(t, body, "Components: "+strings.Join(components, " "))
assert.Contains(t, body, "Architectures: "+strings.Join(architectures, " "))
assert.Contains(t, body, "Components: "+strings.Join(components, " ")+"\n")
assert.Contains(t, body, "Architectures: "+strings.Join(architectures, " ")+"\n")
for _, component := range components {
for _, architecture := range architectures {
assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages", component, architecture))
assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.gz", component, architecture))
assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.xz", component, architecture))
assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages\n", component, architecture))
assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.gz\n", component, architecture))
assert.Contains(t, body, fmt.Sprintf("%s/binary-%s/Packages.xz\n", component, architecture))
}
}
@ -241,6 +247,10 @@ func TestPackageDebian(t *testing.T) {
AddBasicAuthHeader(req, user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "DELETE", fmt.Sprintf("%s/pool/%s/%s/%s/%s/%s", rootURL, distribution, component, packageName, packageVersion2, architecture))
AddBasicAuthHeader(req, user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture))
MakeRequest(t, req, http.StatusNotFound)
}
@ -250,7 +260,7 @@ func TestPackageDebian(t *testing.T) {
body := resp.Body.String()
assert.Contains(t, body, "Components: "+strings.Join(components, " "))
assert.Contains(t, body, "Architectures: "+architectures[1])
assert.Contains(t, body, "Components: "+strings.Join(components, " ")+"\n")
assert.Contains(t, body, "Architectures: "+architectures[1]+"\n")
})
}