diff --git a/docs/content/doc/packages/storage.en-us.md b/docs/content/doc/packages/storage.en-us.md
new file mode 100644
index 000000000..c922496a9
--- /dev/null
+++ b/docs/content/doc/packages/storage.en-us.md
@@ -0,0 +1,84 @@
+---
+date: "2022-11-01T00:00:00+00:00"
+title: "Storage"
+slug: "packages/storage"
+draft: false
+toc: false
+menu:
+ sidebar:
+ parent: "packages"
+ name: "storage"
+ weight: 5
+ identifier: "storage"
+---
+
+# Storage
+
+This document describes the storage of the package registry and how it can be managed.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Deduplication
+
+The package registry has a build-in deduplication of uploaded blobs.
+If two identical files are uploaded only one blob is saved on the filesystem.
+This ensures no space is wasted for duplicated files.
+
+If two packages are uploaded with identical files, both packages will display the same size but on the filesystem they require only half of the size.
+Whenever a package gets deleted only the references to the underlaying blobs are removed.
+The blobs get not removed at this moment, so they still require space on the filesystem.
+When a new package gets uploaded the existing blobs may get referenced again.
+
+These unreferenced blobs get deleted by a [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}).
+The config setting `OLDER_THAN` configures how long unreferenced blobs are kept before they get deleted.
+
+## Cleanup Rules
+
+Package registries can become large over time without cleanup.
+It's recommended to delete unnecessary packages and set up cleanup rules to automatically manage the package registry usage.
+Every package owner (user or organization) manages the cleanup rules which are applied to their packages.
+
+|Setting|Description|
+|-|-|
+|Enabled|Turn the cleanup rule on or off.|
+|Type|Every rule manages a specific package type.|
+|Apply pattern to full package name|If enabled, the patterns below are applied to the full package name (`package/version`). Otherwise only the version (`version`) is used.|
+|Keep the most recent|How many versions to *always* keep for each package.|
+|Keep versions matching|The regex pattern that determines which versions to keep. An empty pattern keeps no version while `.+` keeps all versions. The container registry will always keep the `latest` version even if not configured.|
+|Remove versions older than|Remove only versions older than the selected days.|
+|Remove versions matching|The regex pattern that determines which versions to remove. An empty pattern or `.+` leads to the removal of every package if no other setting tells otherwise.|
+
+Every cleanup rule can show a preview of the affected packages.
+This can be used to check if the cleanup rules is proper configured.
+
+### Regex examples
+
+Regex patterns are automatically surrounded with `\A` and `\z` anchors.
+Do not include any `\A`, `\z`, `^` or `$` token in the regex patterns as they are not necessary.
+The patterns are case-insensitive which matches the behaviour of the package registry in Gitea.
+
+|Pattern|Description|
+|-|-|
+|`.*`|Match every possible version.|
+|`v.+`|Match versions that start with `v`.|
+|`release`|Match only the version `release`.|
+|`release.*`|Match versions that are either named or start with `release`.|
+|`.+-temp-.+`|Match versions that contain `-temp-`.|
+|`v.+\|release`|Match versions that either start with `v` or are named `release`.|
+|`package/v.+\|other/release`|Match versions of the package `package` that start with `v` or the version `release` of the package `other`. This needs the setting *Apply pattern to full package name* enabled.|
+
+### How the cleanup rules work
+
+The cleanup rules are part of the [clean up job]({{< relref "doc/advanced/config-cheat-sheet.en-us.md#cron---cleanup-expired-packages-croncleanup_packages" >}}) and run periodicly.
+
+The cleanup rule:
+
+1. Collects all packages of the package type for the owners registry.
+1. For every package it collects all versions.
+1. Excludes from the list the # versions based on the *Keep the most recent* value.
+1. Excludes from the list any versions matching the *Keep versions matching* value.
+1. Excludes from the list the versions more recent than the *Remove versions older than* value.
+1. Excludes from the list any versions not matching the *Remove versions matching* value.
+1. Deletes the remaining versions.
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index 6ef4ef561..c48fc8d9a 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -439,6 +439,8 @@ var migrations = []Migration{
NewMigration("Alter package_version.metadata_json to LONGTEXT", v1_19.AlterPackageVersionMetadataToLongText),
// v233 -> v234
NewMigration("Add header_authorization_encrypted column to webhook table", v1_19.AddHeaderAuthorizationEncryptedColWebhook),
+ // v234 -> v235
+ NewMigration("Add package cleanup rule table", v1_19.CreatePackageCleanupRuleTable),
}
// GetCurrentDBVersion returns the current db version
diff --git a/models/migrations/v1_19/v234.go b/models/migrations/v1_19/v234.go
new file mode 100644
index 000000000..9d609c58d
--- /dev/null
+++ b/models/migrations/v1_19/v234.go
@@ -0,0 +1,29 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_19 //nolint
+
+import (
+ "code.gitea.io/gitea/modules/timeutil"
+
+ "xorm.io/xorm"
+)
+
+func CreatePackageCleanupRuleTable(x *xorm.Engine) error {
+ type PackageCleanupRule struct {
+ ID int64 `xorm:"pk autoincr"`
+ Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"`
+ OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
+ Type string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+ KeepCount int `xorm:"NOT NULL DEFAULT 0"`
+ KeepPattern string `xorm:"NOT NULL DEFAULT ''"`
+ RemoveDays int `xorm:"NOT NULL DEFAULT 0"`
+ RemovePattern string `xorm:"NOT NULL DEFAULT ''"`
+ MatchFullName bool `xorm:"NOT NULL DEFAULT false"`
+ CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
+ UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
+ }
+
+ return x.Sync2(new(PackageCleanupRule))
+}
diff --git a/models/packages/package.go b/models/packages/package.go
index e39a7c4e4..cea04a095 100644
--- a/models/packages/package.go
+++ b/models/packages/package.go
@@ -45,6 +45,21 @@ const (
TypeVagrant Type = "vagrant"
)
+var TypeList = []Type{
+ TypeComposer,
+ TypeConan,
+ TypeContainer,
+ TypeGeneric,
+ TypeHelm,
+ TypeMaven,
+ TypeNpm,
+ TypeNuGet,
+ TypePub,
+ TypePyPI,
+ TypeRubyGems,
+ TypeVagrant,
+}
+
// Name gets the name of the package type
func (pt Type) Name() string {
switch pt {
diff --git a/models/packages/package_cleanup_rule.go b/models/packages/package_cleanup_rule.go
new file mode 100644
index 000000000..ab45226cf
--- /dev/null
+++ b/models/packages/package_cleanup_rule.go
@@ -0,0 +1,110 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "regexp"
+
+ "code.gitea.io/gitea/models/db"
+ "code.gitea.io/gitea/modules/timeutil"
+
+ "xorm.io/builder"
+)
+
+var ErrPackageCleanupRuleNotExist = errors.New("Package blob does not exist")
+
+func init() {
+ db.RegisterModel(new(PackageCleanupRule))
+}
+
+// PackageCleanupRule represents a rule which describes when to clean up package versions
+type PackageCleanupRule struct {
+ ID int64 `xorm:"pk autoincr"`
+ Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"`
+ OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
+ Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"`
+ KeepCount int `xorm:"NOT NULL DEFAULT 0"`
+ KeepPattern string `xorm:"NOT NULL DEFAULT ''"`
+ KeepPatternMatcher *regexp.Regexp `xorm:"-"`
+ RemoveDays int `xorm:"NOT NULL DEFAULT 0"`
+ RemovePattern string `xorm:"NOT NULL DEFAULT ''"`
+ RemovePatternMatcher *regexp.Regexp `xorm:"-"`
+ MatchFullName bool `xorm:"NOT NULL DEFAULT false"`
+ CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
+ UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
+}
+
+func (pcr *PackageCleanupRule) CompiledPattern() error {
+ if pcr.KeepPatternMatcher != nil || pcr.RemovePatternMatcher != nil {
+ return nil
+ }
+
+ if pcr.KeepPattern != "" {
+ var err error
+ pcr.KeepPatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.KeepPattern))
+ if err != nil {
+ return err
+ }
+ }
+
+ if pcr.RemovePattern != "" {
+ var err error
+ pcr.RemovePatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.RemovePattern))
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) {
+ return pcr, db.Insert(ctx, pcr)
+}
+
+func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) {
+ pcr := &PackageCleanupRule{}
+
+ has, err := db.GetEngine(ctx).ID(id).Get(pcr)
+ if err != nil {
+ return nil, err
+ }
+ if !has {
+ return nil, ErrPackageCleanupRuleNotExist
+ }
+ return pcr, nil
+}
+
+func UpdateCleanupRule(ctx context.Context, pcr *PackageCleanupRule) error {
+ _, err := db.GetEngine(ctx).ID(pcr.ID).AllCols().Update(pcr)
+ return err
+}
+
+func GetCleanupRulesByOwner(ctx context.Context, ownerID int64) ([]*PackageCleanupRule, error) {
+ pcrs := make([]*PackageCleanupRule, 0, 10)
+ return pcrs, db.GetEngine(ctx).Where("owner_id = ?", ownerID).Find(&pcrs)
+}
+
+func DeleteCleanupRuleByID(ctx context.Context, ruleID int64) error {
+ _, err := db.GetEngine(ctx).ID(ruleID).Delete(&PackageCleanupRule{})
+ return err
+}
+
+func HasOwnerCleanupRuleForPackageType(ctx context.Context, ownerID int64, packageType Type) (bool, error) {
+ return db.GetEngine(ctx).
+ Where("owner_id = ? AND type = ?", ownerID, packageType).
+ Exist(&PackageCleanupRule{})
+}
+
+func IterateEnabledCleanupRules(ctx context.Context, callback func(context.Context, *PackageCleanupRule) error) error {
+ return db.Iterate(
+ ctx,
+ builder.Eq{"enabled": true},
+ callback,
+ )
+}
diff --git a/models/packages/package_version.go b/models/packages/package_version.go
index 48c6aa7d6..6ee362502 100644
--- a/models/packages/package_version.go
+++ b/models/packages/package_version.go
@@ -320,6 +320,15 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
return pvs, count, err
}
+// ExistVersion checks if a version matching the search options exist
+func ExistVersion(ctx context.Context, opts *PackageSearchOptions) (bool, error) {
+ return db.GetEngine(ctx).
+ Where(opts.toConds()).
+ Table("package_version").
+ Join("INNER", "package", "package.id = package_version.package_id").
+ Exist(new(PackageVersion))
+}
+
// CountVersions counts all versions of packages matching the search options
func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) {
return db.GetEngine(ctx).
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index eb2a1c86d..ce93e92d3 100644
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -86,6 +86,9 @@ remove = Remove
remove_all = Remove All
edit = Edit
+enabled = Enabled
+disabled = Disabled
+
copy = Copy
copy_url = Copy URL
copy_content = Copy content
@@ -3186,3 +3189,23 @@ settings.delete.description = Deleting a package is permanent and cannot be undo
settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure?
settings.delete.success = The package has been deleted.
settings.delete.error = Failed to delete the package.
+owner.settings.cleanuprules.title = Manage Cleanup Rules
+owner.settings.cleanuprules.add = Add Cleanup Rule
+owner.settings.cleanuprules.edit = Edit Cleanup Rule
+owner.settings.cleanuprules.none = No cleanup rules available. Read the docs to learn more.
+owner.settings.cleanuprules.preview = Cleanup Rule Preview
+owner.settings.cleanuprules.preview.overview = %d packages are scheduled to be removed.
+owner.settings.cleanuprules.preview.none = Cleanup rule does not match any packages.
+owner.settings.cleanuprules.enabled = Enabled
+owner.settings.cleanuprules.pattern_full_match = Apply pattern to full package name
+owner.settings.cleanuprules.keep.title = Versions that match these rules are kept, even if they match a removal rule below.
+owner.settings.cleanuprules.keep.count = Keep the most recent
+owner.settings.cleanuprules.keep.count.1 = 1 version per package
+owner.settings.cleanuprules.keep.count.n = %d versions per package
+owner.settings.cleanuprules.keep.pattern = Keep versions matching
+owner.settings.cleanuprules.keep.pattern.container = The latest
version is always kept for Container packages.
+owner.settings.cleanuprules.remove.title = Versions that match these rules are removed, unless a rule above says to keep them.
+owner.settings.cleanuprules.remove.days = Remove versions older than
+owner.settings.cleanuprules.remove.pattern = Remove versions matching
+owner.settings.cleanuprules.success.update = Cleanup rule has been updated.
+owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted.
diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go
new file mode 100644
index 000000000..c7edf4a18
--- /dev/null
+++ b/routers/web/org/setting_packages.go
@@ -0,0 +1,87 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package org
+
+import (
+ "fmt"
+ "net/http"
+
+ "code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/setting"
+ shared "code.gitea.io/gitea/routers/web/shared/packages"
+)
+
+const (
+ tplSettingsPackages base.TplName = "org/settings/packages"
+ tplSettingsPackagesRuleEdit base.TplName = "org/settings/packages_cleanup_rules_edit"
+ tplSettingsPackagesRulePreview base.TplName = "org/settings/packages_cleanup_rules_preview"
+)
+
+func Packages(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsOrgSettings"] = true
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetPackagesContext(ctx, ctx.ContextUser)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackages)
+}
+
+func PackagesRuleAdd(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsOrgSettings"] = true
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetRuleAddContext(ctx)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
+}
+
+func PackagesRuleEdit(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsOrgSettings"] = true
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetRuleEditContext(ctx, ctx.ContextUser)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
+}
+
+func PackagesRuleAddPost(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsOrgSettings"] = true
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.PerformRuleAddPost(
+ ctx,
+ ctx.ContextUser,
+ fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name),
+ tplSettingsPackagesRuleEdit,
+ )
+}
+
+func PackagesRuleEditPost(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsOrgSettings"] = true
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.PerformRuleEditPost(
+ ctx,
+ ctx.ContextUser,
+ fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name),
+ tplSettingsPackagesRuleEdit,
+ )
+}
+
+func PackagesRulePreview(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsOrgSettings"] = true
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetRulePreviewContext(ctx, ctx.ContextUser)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
+}
diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go
new file mode 100644
index 000000000..5e934d707
--- /dev/null
+++ b/routers/web/shared/packages/packages.go
@@ -0,0 +1,226 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+ "fmt"
+ "net/http"
+ "time"
+
+ "code.gitea.io/gitea/models/db"
+ packages_model "code.gitea.io/gitea/models/packages"
+ user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/util"
+ "code.gitea.io/gitea/modules/web"
+ "code.gitea.io/gitea/services/forms"
+ container_service "code.gitea.io/gitea/services/packages/container"
+)
+
+func SetPackagesContext(ctx *context.Context, owner *user_model.User) {
+ pcrs, err := packages_model.GetCleanupRulesByOwner(ctx, owner.ID)
+ if err != nil {
+ ctx.ServerError("GetCleanupRulesByOwner", err)
+ return
+ }
+
+ ctx.Data["CleanupRules"] = pcrs
+}
+
+func SetRuleAddContext(ctx *context.Context) {
+ setRuleEditContext(ctx, nil)
+}
+
+func SetRuleEditContext(ctx *context.Context, owner *user_model.User) {
+ pcr := getCleanupRuleByContext(ctx, owner)
+ if pcr == nil {
+ return
+ }
+
+ setRuleEditContext(ctx, pcr)
+}
+
+func setRuleEditContext(ctx *context.Context, pcr *packages_model.PackageCleanupRule) {
+ ctx.Data["IsEditRule"] = pcr != nil
+
+ if pcr == nil {
+ pcr = &packages_model.PackageCleanupRule{}
+ }
+ ctx.Data["CleanupRule"] = pcr
+ ctx.Data["AvailableTypes"] = packages_model.TypeList
+}
+
+func PerformRuleAddPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) {
+ performRuleEditPost(ctx, owner, nil, redirectURL, template)
+}
+
+func PerformRuleEditPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) {
+ pcr := getCleanupRuleByContext(ctx, owner)
+ if pcr == nil {
+ return
+ }
+
+ form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm)
+
+ if form.Action == "remove" {
+ if err := packages_model.DeleteCleanupRuleByID(ctx, pcr.ID); err != nil {
+ ctx.ServerError("DeleteCleanupRuleByID", err)
+ return
+ }
+
+ ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.delete"))
+ ctx.Redirect(redirectURL)
+ } else {
+ performRuleEditPost(ctx, owner, pcr, redirectURL, template)
+ }
+}
+
+func performRuleEditPost(ctx *context.Context, owner *user_model.User, pcr *packages_model.PackageCleanupRule, redirectURL string, template base.TplName) {
+ isEditRule := pcr != nil
+
+ if pcr == nil {
+ pcr = &packages_model.PackageCleanupRule{}
+ }
+
+ form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm)
+
+ pcr.Enabled = form.Enabled
+ pcr.OwnerID = owner.ID
+ pcr.KeepCount = form.KeepCount
+ pcr.KeepPattern = form.KeepPattern
+ pcr.RemoveDays = form.RemoveDays
+ pcr.RemovePattern = form.RemovePattern
+ pcr.MatchFullName = form.MatchFullName
+
+ ctx.Data["IsEditRule"] = isEditRule
+ ctx.Data["CleanupRule"] = pcr
+ ctx.Data["AvailableTypes"] = packages_model.TypeList
+
+ if ctx.HasError() {
+ ctx.HTML(http.StatusOK, template)
+ return
+ }
+
+ if isEditRule {
+ if err := packages_model.UpdateCleanupRule(ctx, pcr); err != nil {
+ ctx.ServerError("UpdateCleanupRule", err)
+ return
+ }
+ } else {
+ pcr.Type = packages_model.Type(form.Type)
+
+ if has, err := packages_model.HasOwnerCleanupRuleForPackageType(ctx, owner.ID, pcr.Type); err != nil {
+ ctx.ServerError("HasOwnerCleanupRuleForPackageType", err)
+ return
+ } else if has {
+ ctx.Data["Err_Type"] = true
+ ctx.HTML(http.StatusOK, template)
+ return
+ }
+
+ var err error
+ if pcr, err = packages_model.InsertCleanupRule(ctx, pcr); err != nil {
+ ctx.ServerError("InsertCleanupRule", err)
+ return
+ }
+ }
+
+ ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.update"))
+ ctx.Redirect(fmt.Sprintf("%s/rules/%d", redirectURL, pcr.ID))
+}
+
+func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) {
+ pcr := getCleanupRuleByContext(ctx, owner)
+ if pcr == nil {
+ return
+ }
+
+ if err := pcr.CompiledPattern(); err != nil {
+ ctx.ServerError("CompiledPattern", err)
+ return
+ }
+
+ olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
+
+ packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
+ if err != nil {
+ ctx.ServerError("GetPackagesByType", err)
+ return
+ }
+
+ versionsToRemove := make([]*packages_model.PackageDescriptor, 0, 10)
+
+ for _, p := range packages {
+ pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+ PackageID: p.ID,
+ IsInternal: util.OptionalBoolFalse,
+ Sort: packages_model.SortCreatedDesc,
+ Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
+ })
+ if err != nil {
+ ctx.ServerError("SearchVersions", err)
+ return
+ }
+ for _, pv := range pvs {
+ if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
+ ctx.ServerError("ShouldBeSkipped", err)
+ return
+ } else if skip {
+ continue
+ }
+
+ toMatch := pv.LowerVersion
+ if pcr.MatchFullName {
+ toMatch = p.LowerName + "/" + pv.LowerVersion
+ }
+
+ if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
+ continue
+ }
+ if pv.CreatedUnix.AsLocalTime().After(olderThan) {
+ continue
+ }
+ if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
+ continue
+ }
+
+ pd, err := packages_model.GetPackageDescriptor(ctx, pv)
+ if err != nil {
+ ctx.ServerError("GetPackageDescriptor", err)
+ return
+ }
+ versionsToRemove = append(versionsToRemove, pd)
+ }
+ }
+
+ ctx.Data["CleanupRule"] = pcr
+ ctx.Data["VersionsToRemove"] = versionsToRemove
+}
+
+func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *packages_model.PackageCleanupRule {
+ id := ctx.FormInt64("id")
+ if id == 0 {
+ id = ctx.ParamsInt64("id")
+ }
+
+ pcr, err := packages_model.GetCleanupRuleByID(ctx, id)
+ if err != nil {
+ if err == packages_model.ErrPackageCleanupRuleNotExist {
+ ctx.NotFound("", err)
+ } else {
+ ctx.ServerError("GetCleanupRuleByID", err)
+ }
+ return nil
+ }
+
+ if pcr != nil && pcr.OwnerID == owner.ID {
+ return pcr
+ }
+
+ ctx.NotFound("", fmt.Errorf("PackageCleanupRule[%v] not associated to owner %v", id, owner))
+
+ return nil
+}
diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go
new file mode 100644
index 000000000..d44e90455
--- /dev/null
+++ b/routers/web/user/setting/packages.go
@@ -0,0 +1,80 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+ "net/http"
+
+ "code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/setting"
+ shared "code.gitea.io/gitea/routers/web/shared/packages"
+)
+
+const (
+ tplSettingsPackages base.TplName = "user/settings/packages"
+ tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit"
+ tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview"
+)
+
+func Packages(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetPackagesContext(ctx, ctx.Doer)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackages)
+}
+
+func PackagesRuleAdd(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetRuleAddContext(ctx)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
+}
+
+func PackagesRuleEdit(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetRuleEditContext(ctx, ctx.Doer)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit)
+}
+
+func PackagesRuleAddPost(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("settings")
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.PerformRuleAddPost(
+ ctx,
+ ctx.Doer,
+ setting.AppSubURL+"/user/settings/packages",
+ tplSettingsPackagesRuleEdit,
+ )
+}
+
+func PackagesRuleEditPost(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.PerformRuleEditPost(
+ ctx,
+ ctx.Doer,
+ setting.AppSubURL+"/user/settings/packages",
+ tplSettingsPackagesRuleEdit,
+ )
+}
+
+func PackagesRulePreview(ctx *context.Context) {
+ ctx.Data["Title"] = ctx.Tr("packages.title")
+ ctx.Data["PageIsSettingsPackages"] = true
+
+ shared.SetRulePreviewContext(ctx, ctx.Doer)
+
+ ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
+}
diff --git a/routers/web/web.go b/routers/web/web.go
index 5fefbad88..142f2384e 100644
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -303,6 +303,13 @@ func RegisterRoutes(m *web.Route) {
}
}
+ packagesEnabled := func(ctx *context.Context) {
+ if !setting.Packages.Enabled {
+ ctx.Error(http.StatusForbidden)
+ return
+ }
+ }
+
// FIXME: not all routes need go through same middleware.
// Especially some AJAX requests, we can reduce middleware number to improve performance.
// Routers.
@@ -443,12 +450,27 @@ func RegisterRoutes(m *web.Route) {
m.Combo("/keys").Get(user_setting.Keys).
Post(bindIgnErr(forms.AddKeyForm{}), user_setting.KeysPost)
m.Post("/keys/delete", user_setting.DeleteKey)
+ m.Group("/packages", func() {
+ m.Get("", user_setting.Packages)
+ m.Group("/rules", func() {
+ m.Group("/add", func() {
+ m.Get("", user_setting.PackagesRuleAdd)
+ m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleAddPost)
+ })
+ m.Group("/{id}", func() {
+ m.Get("", user_setting.PackagesRuleEdit)
+ m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleEditPost)
+ m.Get("/preview", user_setting.PackagesRulePreview)
+ })
+ })
+ }, packagesEnabled)
m.Get("/organization", user_setting.Organization)
m.Get("/repos", user_setting.Repos)
m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository)
}, reqSignIn, func(ctx *context.Context) {
ctx.Data["PageIsUserSettings"] = true
ctx.Data["AllThemes"] = setting.UI.Themes
+ ctx.Data["EnablePackages"] = setting.Packages.Enabled
})
m.Group("/user", func() {
@@ -526,12 +548,10 @@ func RegisterRoutes(m *web.Route) {
m.Post("/delete", admin.DeleteRepo)
})
- if setting.Packages.Enabled {
- m.Group("/packages", func() {
- m.Get("", admin.Packages)
- m.Post("/delete", admin.DeletePackageVersion)
- })
- }
+ m.Group("/packages", func() {
+ m.Get("", admin.Packages)
+ m.Post("/delete", admin.DeletePackageVersion)
+ }, packagesEnabled)
m.Group("/hooks", func() {
m.Get("", admin.DefaultOrSystemWebhooks)
@@ -750,8 +770,24 @@ func RegisterRoutes(m *web.Route) {
})
m.Route("/delete", "GET,POST", org.SettingsDelete)
+
+ m.Group("/packages", func() {
+ m.Get("", org.Packages)
+ m.Group("/rules", func() {
+ m.Group("/add", func() {
+ m.Get("", org.PackagesRuleAdd)
+ m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleAddPost)
+ })
+ m.Group("/{id}", func() {
+ m.Get("", org.PackagesRuleEdit)
+ m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleEditPost)
+ m.Get("/preview", org.PackagesRulePreview)
+ })
+ })
+ }, packagesEnabled)
}, func(ctx *context.Context) {
ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable
+ ctx.Data["EnablePackages"] = setting.Packages.Enabled
})
}, context.OrgAssignment(true, true))
}, reqSignIn)
diff --git a/services/forms/package_form.go b/services/forms/package_form.go
new file mode 100644
index 000000000..6c3ff52a9
--- /dev/null
+++ b/services/forms/package_form.go
@@ -0,0 +1,31 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package forms
+
+import (
+ "net/http"
+
+ "code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/web/middleware"
+
+ "gitea.com/go-chi/binding"
+)
+
+type PackageCleanupRuleForm struct {
+ ID int64
+ Enabled bool
+ Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"`
+ KeepCount int `binding:"In(0,1,5,10,25,50,100)"`
+ KeepPattern string `binding:"RegexPattern"`
+ RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`
+ RemovePattern string `binding:"RegexPattern"`
+ MatchFullName bool
+ Action string `binding:"Required;In(save,remove)"`
+}
+
+func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors {
+ ctx := context.GetContext(req)
+ return middleware.Validate(errs, ctx.Data, f, ctx.Locale)
+}
diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go
index d23a481f2..e3d414d45 100644
--- a/services/packages/container/cleanup.go
+++ b/services/packages/container/cleanup.go
@@ -6,13 +6,12 @@ package container
import (
"context"
- "strings"
"time"
packages_model "code.gitea.io/gitea/models/packages"
container_model "code.gitea.io/gitea/models/packages/container"
- user_model "code.gitea.io/gitea/models/user"
container_module "code.gitea.io/gitea/modules/packages/container"
+ "code.gitea.io/gitea/modules/packages/container/oci"
"code.gitea.io/gitea/modules/util"
)
@@ -82,24 +81,30 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e
return nil
}
-// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
-func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
- ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
- if err != nil {
- return err
+func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) {
+ // Always skip the "latest" tag
+ if pv.LowerVersion == "latest" {
+ return true, nil
}
- newOwnerName = strings.ToLower(newOwnerName)
-
- for _, p := range ps {
- if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
- return err
+ // Check if the version is a digest (or untagged)
+ if oci.Digest(pv.LowerVersion).Validate() {
+ // Check if there is another manifest referencing this version
+ has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{
+ PackageID: p.ID,
+ Properties: map[string]string{
+ container_module.PropertyManifestReference: pv.LowerVersion,
+ },
+ })
+ if err != nil {
+ return false, err
}
- if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
- return err
+ // Skip it if the version is referenced
+ if has {
+ return true, nil
}
}
- return nil
+ return false, nil
}
diff --git a/services/packages/container/common.go b/services/packages/container/common.go
new file mode 100644
index 000000000..40d8914a0
--- /dev/null
+++ b/services/packages/container/common.go
@@ -0,0 +1,36 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package container
+
+import (
+ "context"
+ "strings"
+
+ packages_model "code.gitea.io/gitea/models/packages"
+ user_model "code.gitea.io/gitea/models/user"
+ container_module "code.gitea.io/gitea/modules/packages/container"
+)
+
+// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
+func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
+ ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
+ if err != nil {
+ return err
+ }
+
+ newOwnerName = strings.ToLower(newOwnerName)
+
+ for _, p := range ps {
+ if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
+ return err
+ }
+
+ if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/services/packages/packages.go b/services/packages/packages.go
index 76fdd02bf..7343ffc53 100644
--- a/services/packages/packages.go
+++ b/services/packages/packages.go
@@ -443,13 +443,80 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
}
// Cleanup removes expired package data
-func Cleanup(unused context.Context, olderThan time.Duration) error {
- ctx, committer, err := db.TxContext(db.DefaultContext)
+func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
+ ctx, committer, err := db.TxContext(taskCtx)
if err != nil {
return err
}
defer committer.Close()
+ err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
+ select {
+ case <-taskCtx.Done():
+ return db.ErrCancelledf("While processing package cleanup rules")
+ default:
+ }
+
+ if err := pcr.CompiledPattern(); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
+ }
+
+ olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
+
+ packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
+ if err != nil {
+ return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
+ }
+
+ for _, p := range packages {
+ pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+ PackageID: p.ID,
+ IsInternal: util.OptionalBoolFalse,
+ Sort: packages_model.SortCreatedDesc,
+ Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
+ })
+ if err != nil {
+ return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
+ }
+ for _, pv := range pvs {
+ if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
+ } else if skip {
+ log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
+
+ toMatch := pv.LowerVersion
+ if pcr.MatchFullName {
+ toMatch = p.LowerName + "/" + pv.LowerVersion
+ }
+
+ if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
+ log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
+ if pv.CreatedUnix.AsLocalTime().After(olderThan) {
+ log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
+ if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
+ log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
+ continue
+ }
+
+ log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
+
+ if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
+ return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+
if err := container_service.Cleanup(ctx, olderThan); err != nil {
return err
}
diff --git a/templates/org/settings/navbar.tmpl b/templates/org/settings/navbar.tmpl
index e7cbb8734..7df1c8590 100644
--- a/templates/org/settings/navbar.tmpl
+++ b/templates/org/settings/navbar.tmpl
@@ -17,6 +17,11 @@
{{.locale.Tr "settings.applications"}}
{{end}}
+ {{if .EnablePackages}}
+
+ {{.locale.Tr "packages.title"}}
+
+ {{end}}
{{.locale.Tr "org.settings.delete"}}
diff --git a/templates/org/settings/packages.tmpl b/templates/org/settings/packages.tmpl
new file mode 100644
index 000000000..bb5d95e10
--- /dev/null
+++ b/templates/org/settings/packages.tmpl
@@ -0,0 +1,14 @@
+{{template "base/head" .}}
+
{{.locale.Tr "packages.owner.settings.cleanuprules.preview.overview" (len .VersionsToRemove)}}
+{{.locale.Tr "admin.packages.type"}} | +{{.locale.Tr "admin.packages.name"}} | +{{.locale.Tr "admin.packages.version"}} | +{{.locale.Tr "admin.packages.creator"}} | +{{.locale.Tr "admin.packages.size"}} | +{{.locale.Tr "admin.packages.published"}} | +
---|---|---|---|---|---|
{{.Package.Type.Name}} | +{{.Package.Name}} | +{{.Version.Version}} | +{{.Creator.Name}} | +{{FileSize .CalculateBlobSize}} | ++ |
{{.locale.Tr "packages.owner.settings.cleanuprules.preview.none"}} | +