diff --git a/docs/content/doc/packages/storage.en-us.md b/docs/content/doc/packages/storage.en-us.md new file mode 100644 index 000000000..c922496a9 --- /dev/null +++ b/docs/content/doc/packages/storage.en-us.md @@ -0,0 +1,84 @@ +--- +date: "2022-11-01T00:00:00+00:00" +title: "Storage" +slug: "packages/storage" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "storage" + weight: 5 + identifier: "storage" +--- + +# Storage + +This document describes the storage of the package registry and how it can be managed. + +**Table of Contents** + +{{< toc >}} + +## Deduplication + +The package registry has a build-in deduplication of uploaded blobs. +If two identical files are uploaded only one blob is saved on the filesystem. +This ensures no space is wasted for duplicated files. + +If two packages are uploaded with identical files, both packages will display the same size but on the filesystem they require only half of the size. +Whenever a package gets deleted only the references to the underlaying blobs are removed. +The blobs get not removed at this moment, so they still require space on the filesystem. +When a new package gets uploaded the existing blobs may get referenced again. + +These unreferenced blobs get deleted by a [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}). +The config setting `OLDER_THAN` configures how long unreferenced blobs are kept before they get deleted. + +## Cleanup Rules + +Package registries can become large over time without cleanup. +It's recommended to delete unnecessary packages and set up cleanup rules to automatically manage the package registry usage. +Every package owner (user or organization) manages the cleanup rules which are applied to their packages. + +|Setting|Description| +|-|-| +|Enabled|Turn the cleanup rule on or off.| +|Type|Every rule manages a specific package type.| +|Apply pattern to full package name|If enabled, the patterns below are applied to the full package name (`package/version`). Otherwise only the version (`version`) is used.| +|Keep the most recent|How many versions to *always* keep for each package.| +|Keep versions matching|The regex pattern that determines which versions to keep. An empty pattern keeps no version while `.+` keeps all versions. The container registry will always keep the `latest` version even if not configured.| +|Remove versions older than|Remove only versions older than the selected days.| +|Remove versions matching|The regex pattern that determines which versions to remove. An empty pattern or `.+` leads to the removal of every package if no other setting tells otherwise.| + +Every cleanup rule can show a preview of the affected packages. +This can be used to check if the cleanup rules is proper configured. + +### Regex examples + +Regex patterns are automatically surrounded with `\A` and `\z` anchors. +Do not include any `\A`, `\z`, `^` or `$` token in the regex patterns as they are not necessary. +The patterns are case-insensitive which matches the behaviour of the package registry in Gitea. + +|Pattern|Description| +|-|-| +|`.*`|Match every possible version.| +|`v.+`|Match versions that start with `v`.| +|`release`|Match only the version `release`.| +|`release.*`|Match versions that are either named or start with `release`.| +|`.+-temp-.+`|Match versions that contain `-temp-`.| +|`v.+\|release`|Match versions that either start with `v` or are named `release`.| +|`package/v.+\|other/release`|Match versions of the package `package` that start with `v` or the version `release` of the package `other`. This needs the setting *Apply pattern to full package name* enabled.| + +### How the cleanup rules work + +The cleanup rules are part of the [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}) and run periodicly. + +The cleanup rule: + +1. Collects all packages of the package type for the owners registry. +1. For every package it collects all versions. +1. Excludes from the list the # versions based on the *Keep the most recent* value. +1. Excludes from the list any versions matching the *Keep versions matching* value. +1. Excludes from the list the versions more recent than the *Remove versions older than* value. +1. Excludes from the list any versions not matching the *Remove versions matching* value. +1. Deletes the remaining versions. diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 6ef4ef561..c48fc8d9a 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -439,6 +439,8 @@ var migrations = []Migration{ NewMigration("Alter package_version.metadata_json to LONGTEXT", v1_19.AlterPackageVersionMetadataToLongText), // v233 -> v234 NewMigration("Add header_authorization_encrypted column to webhook table", v1_19.AddHeaderAuthorizationEncryptedColWebhook), + // v234 -> v235 + NewMigration("Add package cleanup rule table", v1_19.CreatePackageCleanupRuleTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_19/v234.go b/models/migrations/v1_19/v234.go new file mode 100644 index 000000000..9d609c58d --- /dev/null +++ b/models/migrations/v1_19/v234.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package v1_19 //nolint + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func CreatePackageCleanupRuleTable(x *xorm.Engine) error { + type PackageCleanupRule struct { + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(PackageCleanupRule)) +} diff --git a/models/packages/package.go b/models/packages/package.go index e39a7c4e4..cea04a095 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -45,6 +45,21 @@ const ( TypeVagrant Type = "vagrant" ) +var TypeList = []Type{ + TypeComposer, + TypeConan, + TypeContainer, + TypeGeneric, + TypeHelm, + TypeMaven, + TypeNpm, + TypeNuGet, + TypePub, + TypePyPI, + TypeRubyGems, + TypeVagrant, +} + // Name gets the name of the package type func (pt Type) Name() string { switch pt { diff --git a/models/packages/package_cleanup_rule.go b/models/packages/package_cleanup_rule.go new file mode 100644 index 000000000..ab45226cf --- /dev/null +++ b/models/packages/package_cleanup_rule.go @@ -0,0 +1,110 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + "fmt" + "regexp" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +var ErrPackageCleanupRuleNotExist = errors.New("Package blob does not exist") + +func init() { + db.RegisterModel(new(PackageCleanupRule)) +} + +// PackageCleanupRule represents a rule which describes when to clean up package versions +type PackageCleanupRule struct { + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + KeepPatternMatcher *regexp.Regexp `xorm:"-"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + RemovePatternMatcher *regexp.Regexp `xorm:"-"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` +} + +func (pcr *PackageCleanupRule) CompiledPattern() error { + if pcr.KeepPatternMatcher != nil || pcr.RemovePatternMatcher != nil { + return nil + } + + if pcr.KeepPattern != "" { + var err error + pcr.KeepPatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.KeepPattern)) + if err != nil { + return err + } + } + + if pcr.RemovePattern != "" { + var err error + pcr.RemovePatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.RemovePattern)) + if err != nil { + return err + } + } + + return nil +} + +func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) { + return pcr, db.Insert(ctx, pcr) +} + +func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) { + pcr := &PackageCleanupRule{} + + has, err := db.GetEngine(ctx).ID(id).Get(pcr) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageCleanupRuleNotExist + } + return pcr, nil +} + +func UpdateCleanupRule(ctx context.Context, pcr *PackageCleanupRule) error { + _, err := db.GetEngine(ctx).ID(pcr.ID).AllCols().Update(pcr) + return err +} + +func GetCleanupRulesByOwner(ctx context.Context, ownerID int64) ([]*PackageCleanupRule, error) { + pcrs := make([]*PackageCleanupRule, 0, 10) + return pcrs, db.GetEngine(ctx).Where("owner_id = ?", ownerID).Find(&pcrs) +} + +func DeleteCleanupRuleByID(ctx context.Context, ruleID int64) error { + _, err := db.GetEngine(ctx).ID(ruleID).Delete(&PackageCleanupRule{}) + return err +} + +func HasOwnerCleanupRuleForPackageType(ctx context.Context, ownerID int64, packageType Type) (bool, error) { + return db.GetEngine(ctx). + Where("owner_id = ? AND type = ?", ownerID, packageType). + Exist(&PackageCleanupRule{}) +} + +func IterateEnabledCleanupRules(ctx context.Context, callback func(context.Context, *PackageCleanupRule) error) error { + return db.Iterate( + ctx, + builder.Eq{"enabled": true}, + callback, + ) +} diff --git a/models/packages/package_version.go b/models/packages/package_version.go index 48c6aa7d6..6ee362502 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -320,6 +320,15 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P return pvs, count, err } +// ExistVersion checks if a version matching the search options exist +func ExistVersion(ctx context.Context, opts *PackageSearchOptions) (bool, error) { + return db.GetEngine(ctx). + Where(opts.toConds()). + Table("package_version"). + Join("INNER", "package", "package.id = package_version.package_id"). + Exist(new(PackageVersion)) +} + // CountVersions counts all versions of packages matching the search options func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) { return db.GetEngine(ctx). diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index eb2a1c86d..ce93e92d3 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -86,6 +86,9 @@ remove = Remove remove_all = Remove All edit = Edit +enabled = Enabled +disabled = Disabled + copy = Copy copy_url = Copy URL copy_content = Copy content @@ -3186,3 +3189,23 @@ settings.delete.description = Deleting a package is permanent and cannot be undo settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? settings.delete.success = The package has been deleted. settings.delete.error = Failed to delete the package. +owner.settings.cleanuprules.title = Manage Cleanup Rules +owner.settings.cleanuprules.add = Add Cleanup Rule +owner.settings.cleanuprules.edit = Edit Cleanup Rule +owner.settings.cleanuprules.none = No cleanup rules available. Read the docs to learn more. +owner.settings.cleanuprules.preview = Cleanup Rule Preview +owner.settings.cleanuprules.preview.overview = %d packages are scheduled to be removed. +owner.settings.cleanuprules.preview.none = Cleanup rule does not match any packages. +owner.settings.cleanuprules.enabled = Enabled +owner.settings.cleanuprules.pattern_full_match = Apply pattern to full package name +owner.settings.cleanuprules.keep.title = Versions that match these rules are kept, even if they match a removal rule below. +owner.settings.cleanuprules.keep.count = Keep the most recent +owner.settings.cleanuprules.keep.count.1 = 1 version per package +owner.settings.cleanuprules.keep.count.n = %d versions per package +owner.settings.cleanuprules.keep.pattern = Keep versions matching +owner.settings.cleanuprules.keep.pattern.container = The latest version is always kept for Container packages. +owner.settings.cleanuprules.remove.title = Versions that match these rules are removed, unless a rule above says to keep them. +owner.settings.cleanuprules.remove.days = Remove versions older than +owner.settings.cleanuprules.remove.pattern = Remove versions matching +owner.settings.cleanuprules.success.update = Cleanup rule has been updated. +owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted. diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go new file mode 100644 index 000000000..c7edf4a18 --- /dev/null +++ b/routers/web/org/setting_packages.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package org + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + shared "code.gitea.io/gitea/routers/web/shared/packages" +) + +const ( + tplSettingsPackages base.TplName = "org/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "org/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "org/settings/packages_cleanup_rules_preview" +) + +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetPackagesContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackages) +} + +func PackagesRuleAdd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleAddContext(ctx) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleEdit(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleEditContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleAddPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleAddPost( + ctx, + ctx.ContextUser, + fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name), + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRuleEditPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleEditPost( + ctx, + ctx.ContextUser, + fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name), + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRulePreview(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRulePreviewContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go new file mode 100644 index 000000000..5e934d707 --- /dev/null +++ b/routers/web/shared/packages/packages.go @@ -0,0 +1,226 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "net/http" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/forms" + container_service "code.gitea.io/gitea/services/packages/container" +) + +func SetPackagesContext(ctx *context.Context, owner *user_model.User) { + pcrs, err := packages_model.GetCleanupRulesByOwner(ctx, owner.ID) + if err != nil { + ctx.ServerError("GetCleanupRulesByOwner", err) + return + } + + ctx.Data["CleanupRules"] = pcrs +} + +func SetRuleAddContext(ctx *context.Context) { + setRuleEditContext(ctx, nil) +} + +func SetRuleEditContext(ctx *context.Context, owner *user_model.User) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + setRuleEditContext(ctx, pcr) +} + +func setRuleEditContext(ctx *context.Context, pcr *packages_model.PackageCleanupRule) { + ctx.Data["IsEditRule"] = pcr != nil + + if pcr == nil { + pcr = &packages_model.PackageCleanupRule{} + } + ctx.Data["CleanupRule"] = pcr + ctx.Data["AvailableTypes"] = packages_model.TypeList +} + +func PerformRuleAddPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) { + performRuleEditPost(ctx, owner, nil, redirectURL, template) +} + +func PerformRuleEditPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm) + + if form.Action == "remove" { + if err := packages_model.DeleteCleanupRuleByID(ctx, pcr.ID); err != nil { + ctx.ServerError("DeleteCleanupRuleByID", err) + return + } + + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.delete")) + ctx.Redirect(redirectURL) + } else { + performRuleEditPost(ctx, owner, pcr, redirectURL, template) + } +} + +func performRuleEditPost(ctx *context.Context, owner *user_model.User, pcr *packages_model.PackageCleanupRule, redirectURL string, template base.TplName) { + isEditRule := pcr != nil + + if pcr == nil { + pcr = &packages_model.PackageCleanupRule{} + } + + form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm) + + pcr.Enabled = form.Enabled + pcr.OwnerID = owner.ID + pcr.KeepCount = form.KeepCount + pcr.KeepPattern = form.KeepPattern + pcr.RemoveDays = form.RemoveDays + pcr.RemovePattern = form.RemovePattern + pcr.MatchFullName = form.MatchFullName + + ctx.Data["IsEditRule"] = isEditRule + ctx.Data["CleanupRule"] = pcr + ctx.Data["AvailableTypes"] = packages_model.TypeList + + if ctx.HasError() { + ctx.HTML(http.StatusOK, template) + return + } + + if isEditRule { + if err := packages_model.UpdateCleanupRule(ctx, pcr); err != nil { + ctx.ServerError("UpdateCleanupRule", err) + return + } + } else { + pcr.Type = packages_model.Type(form.Type) + + if has, err := packages_model.HasOwnerCleanupRuleForPackageType(ctx, owner.ID, pcr.Type); err != nil { + ctx.ServerError("HasOwnerCleanupRuleForPackageType", err) + return + } else if has { + ctx.Data["Err_Type"] = true + ctx.HTML(http.StatusOK, template) + return + } + + var err error + if pcr, err = packages_model.InsertCleanupRule(ctx, pcr); err != nil { + ctx.ServerError("InsertCleanupRule", err) + return + } + } + + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.update")) + ctx.Redirect(fmt.Sprintf("%s/rules/%d", redirectURL, pcr.ID)) +} + +func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + if err := pcr.CompiledPattern(); err != nil { + ctx.ServerError("CompiledPattern", err) + return + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + ctx.ServerError("GetPackagesByType", err) + return + } + + versionsToRemove := make([]*packages_model.PackageDescriptor, 0, 10) + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + ctx.ServerError("SearchVersions", err) + return + } + for _, pv := range pvs { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + ctx.ServerError("ShouldBeSkipped", err) + return + } else if skip { + continue + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + continue + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + ctx.ServerError("GetPackageDescriptor", err) + return + } + versionsToRemove = append(versionsToRemove, pd) + } + } + + ctx.Data["CleanupRule"] = pcr + ctx.Data["VersionsToRemove"] = versionsToRemove +} + +func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *packages_model.PackageCleanupRule { + id := ctx.FormInt64("id") + if id == 0 { + id = ctx.ParamsInt64("id") + } + + pcr, err := packages_model.GetCleanupRuleByID(ctx, id) + if err != nil { + if err == packages_model.ErrPackageCleanupRuleNotExist { + ctx.NotFound("", err) + } else { + ctx.ServerError("GetCleanupRuleByID", err) + } + return nil + } + + if pcr != nil && pcr.OwnerID == owner.ID { + return pcr + } + + ctx.NotFound("", fmt.Errorf("PackageCleanupRule[%v] not associated to owner %v", id, owner)) + + return nil +} diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go new file mode 100644 index 000000000..d44e90455 --- /dev/null +++ b/routers/web/user/setting/packages.go @@ -0,0 +1,80 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "net/http" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + shared "code.gitea.io/gitea/routers/web/shared/packages" +) + +const ( + tplSettingsPackages base.TplName = "user/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview" +) + +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetPackagesContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackages) +} + +func PackagesRuleAdd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleAddContext(ctx) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleEdit(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleEditContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleAddPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("settings") + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleAddPost( + ctx, + ctx.Doer, + setting.AppSubURL+"/user/settings/packages", + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRuleEditPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleEditPost( + ctx, + ctx.Doer, + setting.AppSubURL+"/user/settings/packages", + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRulePreview(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRulePreviewContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) +} diff --git a/routers/web/web.go b/routers/web/web.go index 5fefbad88..142f2384e 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -303,6 +303,13 @@ func RegisterRoutes(m *web.Route) { } } + packagesEnabled := func(ctx *context.Context) { + if !setting.Packages.Enabled { + ctx.Error(http.StatusForbidden) + return + } + } + // FIXME: not all routes need go through same middleware. // Especially some AJAX requests, we can reduce middleware number to improve performance. // Routers. @@ -443,12 +450,27 @@ func RegisterRoutes(m *web.Route) { m.Combo("/keys").Get(user_setting.Keys). Post(bindIgnErr(forms.AddKeyForm{}), user_setting.KeysPost) m.Post("/keys/delete", user_setting.DeleteKey) + m.Group("/packages", func() { + m.Get("", user_setting.Packages) + m.Group("/rules", func() { + m.Group("/add", func() { + m.Get("", user_setting.PackagesRuleAdd) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleAddPost) + }) + m.Group("/{id}", func() { + m.Get("", user_setting.PackagesRuleEdit) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleEditPost) + m.Get("/preview", user_setting.PackagesRulePreview) + }) + }) + }, packagesEnabled) m.Get("/organization", user_setting.Organization) m.Get("/repos", user_setting.Repos) m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository) }, reqSignIn, func(ctx *context.Context) { ctx.Data["PageIsUserSettings"] = true ctx.Data["AllThemes"] = setting.UI.Themes + ctx.Data["EnablePackages"] = setting.Packages.Enabled }) m.Group("/user", func() { @@ -526,12 +548,10 @@ func RegisterRoutes(m *web.Route) { m.Post("/delete", admin.DeleteRepo) }) - if setting.Packages.Enabled { - m.Group("/packages", func() { - m.Get("", admin.Packages) - m.Post("/delete", admin.DeletePackageVersion) - }) - } + m.Group("/packages", func() { + m.Get("", admin.Packages) + m.Post("/delete", admin.DeletePackageVersion) + }, packagesEnabled) m.Group("/hooks", func() { m.Get("", admin.DefaultOrSystemWebhooks) @@ -750,8 +770,24 @@ func RegisterRoutes(m *web.Route) { }) m.Route("/delete", "GET,POST", org.SettingsDelete) + + m.Group("/packages", func() { + m.Get("", org.Packages) + m.Group("/rules", func() { + m.Group("/add", func() { + m.Get("", org.PackagesRuleAdd) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleAddPost) + }) + m.Group("/{id}", func() { + m.Get("", org.PackagesRuleEdit) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleEditPost) + m.Get("/preview", org.PackagesRulePreview) + }) + }) + }, packagesEnabled) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable + ctx.Data["EnablePackages"] = setting.Packages.Enabled }) }, context.OrgAssignment(true, true)) }, reqSignIn) diff --git a/services/forms/package_form.go b/services/forms/package_form.go new file mode 100644 index 000000000..6c3ff52a9 --- /dev/null +++ b/services/forms/package_form.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package forms + +import ( + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/web/middleware" + + "gitea.com/go-chi/binding" +) + +type PackageCleanupRuleForm struct { + ID int64 + Enabled bool + Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + KeepCount int `binding:"In(0,1,5,10,25,50,100)"` + KeepPattern string `binding:"RegexPattern"` + RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` + RemovePattern string `binding:"RegexPattern"` + MatchFullName bool + Action string `binding:"Required;In(save,remove)"` +} + +func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index d23a481f2..e3d414d45 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -6,13 +6,12 @@ package container import ( "context" - "strings" "time" packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" - user_model "code.gitea.io/gitea/models/user" container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/util" ) @@ -82,24 +81,30 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e return nil } -// UpdateRepositoryNames updates the repository name property for all packages of the specific owner -func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error { - ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer) - if err != nil { - return err +func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) { + // Always skip the "latest" tag + if pv.LowerVersion == "latest" { + return true, nil } - newOwnerName = strings.ToLower(newOwnerName) - - for _, p := range ps { - if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil { - return err + // Check if the version is a digest (or untagged) + if oci.Digest(pv.LowerVersion).Validate() { + // Check if there is another manifest referencing this version + has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + Properties: map[string]string{ + container_module.PropertyManifestReference: pv.LowerVersion, + }, + }) + if err != nil { + return false, err } - if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil { - return err + // Skip it if the version is referenced + if has { + return true, nil } } - return nil + return false, nil } diff --git a/services/packages/container/common.go b/services/packages/container/common.go new file mode 100644 index 000000000..40d8914a0 --- /dev/null +++ b/services/packages/container/common.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + container_module "code.gitea.io/gitea/modules/packages/container" +) + +// UpdateRepositoryNames updates the repository name property for all packages of the specific owner +func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error { + ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer) + if err != nil { + return err + } + + newOwnerName = strings.ToLower(newOwnerName) + + for _, p := range ps { + if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil { + return err + } + + if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil { + return err + } + } + + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 76fdd02bf..7343ffc53 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -443,13 +443,80 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro } // Cleanup removes expired package data -func Cleanup(unused context.Context, olderThan time.Duration) error { - ctx, committer, err := db.TxContext(db.DefaultContext) +func Cleanup(taskCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(taskCtx) if err != nil { return err } defer committer.Close() + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-taskCtx.Done(): + return db.ErrCancelledf("While processing package cleanup rules") + default: + } + + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + for _, pv := range pvs { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + } + } + return nil + }) + if err != nil { + return err + } + if err := container_service.Cleanup(ctx, olderThan); err != nil { return err } diff --git a/templates/org/settings/navbar.tmpl b/templates/org/settings/navbar.tmpl index e7cbb8734..7df1c8590 100644 --- a/templates/org/settings/navbar.tmpl +++ b/templates/org/settings/navbar.tmpl @@ -17,6 +17,11 @@ {{.locale.Tr "settings.applications"}} {{end}} + {{if .EnablePackages}} + + {{.locale.Tr "packages.title"}} + + {{end}} {{.locale.Tr "org.settings.delete"}} diff --git a/templates/org/settings/packages.tmpl b/templates/org/settings/packages.tmpl new file mode 100644 index 000000000..bb5d95e10 --- /dev/null +++ b/templates/org/settings/packages.tmpl @@ -0,0 +1,14 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/list" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/org/settings/packages_cleanup_rules_edit.tmpl b/templates/org/settings/packages_cleanup_rules_edit.tmpl new file mode 100644 index 000000000..8c3725f4d --- /dev/null +++ b/templates/org/settings/packages_cleanup_rules_edit.tmpl @@ -0,0 +1,14 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/edit" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/org/settings/packages_cleanup_rules_preview.tmpl b/templates/org/settings/packages_cleanup_rules_preview.tmpl new file mode 100644 index 000000000..e0e4652c3 --- /dev/null +++ b/templates/org/settings/packages_cleanup_rules_preview.tmpl @@ -0,0 +1,13 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "package/shared/cleanup_rules/preview" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/package/shared/cleanup_rules/edit.tmpl b/templates/package/shared/cleanup_rules/edit.tmpl new file mode 100644 index 000000000..f8525afb7 --- /dev/null +++ b/templates/package/shared/cleanup_rules/edit.tmpl @@ -0,0 +1,73 @@ +

{{if .IsEditRule}}{{.locale.Tr "packages.owner.settings.cleanuprules.edit"}}{{else}}{{.locale.Tr "packages.owner.settings.cleanuprules.add"}}{{end}}

+
+
+ {{.CsrfTokenHtml}} + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.keep.title"}}

+
+ + +
+
+ + +

{{.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern.container" | Safe}}

+
+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.remove.title"}}

+
+ + +
+
+ + +
+
+ {{if .IsEditRule}} + + + {{.locale.Tr "packages.owner.settings.cleanuprules.preview"}} + {{else}} + + {{end}} +
+
+
diff --git a/templates/package/shared/cleanup_rules/list.tmpl b/templates/package/shared/cleanup_rules/list.tmpl new file mode 100644 index 000000000..09f95e4f4 --- /dev/null +++ b/templates/package/shared/cleanup_rules/list.tmpl @@ -0,0 +1,34 @@ +

+ {{.locale.Tr "packages.owner.settings.cleanuprules.title"}} +
+ {{.locale.Tr "packages.owner.settings.cleanuprules.add"}} +
+

+
+
+ {{range .CleanupRules}} +
+ + {{svg .Type.SVGName 36}} +
+ {{.Type.Name}} +
{{if .Enabled}}{{$.locale.Tr "enabled"}}{{else}}{{$.locale.Tr "disabled"}}{{end}}
+ {{if .KeepCount}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count"}}: {{if eq .KeepCount 1}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.1"}}{{else}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" .KeepCount}}{{end}}
{{end}} + {{if .KeepPattern}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern"}}: {{EllipsisString .KeepPattern 100}}
{{end}} + {{if .RemoveDays}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.days"}}: {{$.locale.Tr "tool.days" .RemoveDays}}
{{end}} + {{if .RemovePattern}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.pattern"}}: {{EllipsisString .RemovePattern 100}}
{{end}} +
+
+ {{else}} +
{{.locale.Tr "packages.owner.settings.cleanuprules.none"}}
+ {{end}} +
+
diff --git a/templates/package/shared/cleanup_rules/preview.tmpl b/templates/package/shared/cleanup_rules/preview.tmpl new file mode 100644 index 000000000..c59ad67f7 --- /dev/null +++ b/templates/package/shared/cleanup_rules/preview.tmpl @@ -0,0 +1,34 @@ +

{{.locale.Tr "packages.owner.settings.cleanuprules.preview"}}

+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.preview.overview" (len .VersionsToRemove)}}

+
+
+ + + + + + + + + + + + + {{range .VersionsToRemove}} + + + + + + + + + {{else}} + + + + {{end}} + +
{{.locale.Tr "admin.packages.type"}}{{.locale.Tr "admin.packages.name"}}{{.locale.Tr "admin.packages.version"}}{{.locale.Tr "admin.packages.creator"}}{{.locale.Tr "admin.packages.size"}}{{.locale.Tr "admin.packages.published"}}
{{.Package.Type.Name}}{{.Package.Name}}{{.Version.Version}}{{.Creator.Name}}{{FileSize .CalculateBlobSize}}
{{.locale.Tr "packages.owner.settings.cleanuprules.preview.none"}}
+
diff --git a/templates/user/settings/navbar.tmpl b/templates/user/settings/navbar.tmpl index 01ae055d7..d17494fc0 100644 --- a/templates/user/settings/navbar.tmpl +++ b/templates/user/settings/navbar.tmpl @@ -18,6 +18,11 @@ {{.locale.Tr "settings.ssh_gpg_keys"}} + {{if .EnablePackages}} + + {{.locale.Tr "packages.title"}} + + {{end}} {{.locale.Tr "settings.organization"}} diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl new file mode 100644 index 000000000..261231345 --- /dev/null +++ b/templates/user/settings/packages.tmpl @@ -0,0 +1,9 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/list" .}} +
+
+{{template "base/footer" .}} diff --git a/templates/user/settings/packages_cleanup_rules_edit.tmpl b/templates/user/settings/packages_cleanup_rules_edit.tmpl new file mode 100644 index 000000000..4cf642b7e --- /dev/null +++ b/templates/user/settings/packages_cleanup_rules_edit.tmpl @@ -0,0 +1,9 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/edit" .}} +
+
+{{template "base/footer" .}} diff --git a/templates/user/settings/packages_cleanup_rules_preview.tmpl b/templates/user/settings/packages_cleanup_rules_preview.tmpl new file mode 100644 index 000000000..20041f9a4 --- /dev/null +++ b/templates/user/settings/packages_cleanup_rules_preview.tmpl @@ -0,0 +1,8 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "package/shared/cleanup_rules/preview" .}} +
+
+{{template "base/footer" .}} diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 815685ea7..8efb70848 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -203,22 +203,171 @@ func TestPackageQuota(t *testing.T) { func TestPackageCleanup(t *testing.T) { defer tests.PrepareTestEnv(t)() - time.Sleep(time.Second) + duration, _ := time.ParseDuration("-1h") - pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0)) - assert.NoError(t, err) - assert.NotEmpty(t, pbs) + t.Run("Common", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) - assert.NoError(t, err) + pbs, err := packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) + assert.NoError(t, err) + assert.NotEmpty(t, pbs) - err = packages_service.Cleanup(nil, time.Duration(0)) - assert.NoError(t, err) + _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) + assert.NoError(t, err) - pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, time.Duration(0)) - assert.NoError(t, err) - assert.Empty(t, pbs) + err = packages_service.Cleanup(db.DefaultContext, duration) + assert.NoError(t, err) - _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) - assert.ErrorIs(t, err, packages_model.ErrPackageNotExist) + pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) + assert.NoError(t, err) + assert.Empty(t, pbs) + + _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) + assert.ErrorIs(t, err, packages_model.ErrPackageNotExist) + }) + + t.Run("CleanupRules", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + type version struct { + Version string + ShouldExist bool + Created int64 + } + + cases := []struct { + Name string + Versions []version + Rule *packages_model.PackageCleanupRule + }{ + { + Name: "Disabled", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: false, + }, + }, + { + Name: "KeepCount", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "v1.0", ShouldExist: true}, + {Version: "test-3", ShouldExist: false, Created: 1}, + {Version: "test-4", ShouldExist: false, Created: 1}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + KeepCount: 2, + }, + }, + { + Name: "KeepPattern", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "v1.0", ShouldExist: false}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + KeepPattern: "k.+p", + }, + }, + { + Name: "RemoveDays", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "v1.0", ShouldExist: false, Created: 1}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + RemoveDays: 60, + }, + }, + { + Name: "RemovePattern", + Versions: []version{ + {Version: "test", ShouldExist: true}, + {Version: "test-3", ShouldExist: false}, + {Version: "test-4", ShouldExist: false}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + RemovePattern: `t[e]+st-\d+`, + }, + }, + { + Name: "MatchFullName", + Versions: []version{ + {Version: "keep", ShouldExist: true}, + {Version: "test", ShouldExist: false}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + RemovePattern: `package/test|different/keep`, + MatchFullName: true, + }, + }, + { + Name: "Mixed", + Versions: []version{ + {Version: "keep", ShouldExist: true, Created: time.Now().Add(time.Duration(10000)).Unix()}, + {Version: "dummy", ShouldExist: true, Created: 1}, + {Version: "test-3", ShouldExist: true}, + {Version: "test-4", ShouldExist: false, Created: 1}, + }, + Rule: &packages_model.PackageCleanupRule{ + Enabled: true, + KeepCount: 1, + KeepPattern: `dummy`, + RemoveDays: 7, + RemovePattern: `t[e]+st-\d+`, + }, + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + for _, v := range c.Versions { + url := fmt.Sprintf("/api/packages/%s/generic/package/%s/file.bin", user.Name, v.Version) + req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{1})) + AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusCreated) + + if v.Created != 0 { + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeGeneric, "package", v.Version) + assert.NoError(t, err) + _, err = db.GetEngine(db.DefaultContext).Exec("UPDATE package_version SET created_unix = ? WHERE id = ?", v.Created, pv.ID) + assert.NoError(t, err) + } + } + + c.Rule.OwnerID = user.ID + c.Rule.Type = packages_model.TypeGeneric + + pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) + assert.NoError(t, err) + + err = packages_service.Cleanup(db.DefaultContext, duration) + assert.NoError(t, err) + + for _, v := range c.Versions { + pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages_model.TypeGeneric, "package", v.Version) + if v.ShouldExist { + assert.NoError(t, err) + err = packages_service.DeletePackageVersionAndReferences(db.DefaultContext, pv) + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, packages_model.ErrPackageNotExist) + } + } + + assert.NoError(t, packages_model.DeleteCleanupRuleByID(db.DefaultContext, pcr.ID)) + }) + } + }) }