From d0d6aad85f4d1e2a6d2a6524fe13eccecfd350af Mon Sep 17 00:00:00 2001 From: dicarne Date: Wed, 15 May 2024 21:56:17 +0800 Subject: [PATCH] Supports forced use of S3 virtual-hosted style (#30969) Add a configuration item to enable S3 virtual-hosted style (V2) to solve the problem caused by some S3 service providers not supporting path style (V1). --- cmd/migrate_storage.go | 6 ++++++ custom/conf/app.example.ini | 6 ++++++ .../config-cheat-sheet.en-us.md | 8 ++++++++ .../config-cheat-sheet.zh-cn.md | 8 ++++++++ modules/setting/storage.go | 2 ++ modules/storage/minio.go | 20 +++++++++++++++---- 6 files changed, 46 insertions(+), 4 deletions(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 357416fc3..7d1ef052f 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -91,6 +91,11 @@ var CmdMigrateStorage = &cli.Command{ Value: "", Usage: "Minio checksum algorithm (default/md5)", }, + &cli.StringFlag{ + Name: "minio-bucket-lookup-type", + Value: "", + Usage: "Minio bucket lookup type", + }, }, } @@ -220,6 +225,7 @@ func runMigrateStorage(ctx *cli.Context) error { UseSSL: ctx.Bool("minio-use-ssl"), InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"), ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"), + BucketLookUpType: ctx.String("minio-bucket-lookup-type"), }, }) default: diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 577479e39..4df843b8c 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1895,6 +1895,9 @@ LEVEL = Info ;; ;; Minio checksum algorithm: default (for MinIO or AWS S3) or md5 (for Cloudflare or Backblaze) ;MINIO_CHECKSUM_ALGORITHM = default +;; +;; Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` +;MINIO_BUCKET_LOOKUP_TYPE = auto ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2576,6 +2579,9 @@ LEVEL = Info ;; ;; Minio skip SSL verification available when STORAGE_TYPE is `minio` ;MINIO_INSECURE_SKIP_VERIFY = false +;; +;; Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` +;MINIO_BUCKET_LOOKUP_TYPE = auto ;[proxy] ;; Enable the proxy, all requests to external via HTTP will be affected diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index 07712c111..6c429bb65 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -851,6 +851,7 @@ Default templates for project boards: - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` - `MINIO_CHECKSUM_ALGORITHM`: **default**: Minio checksum algorithm: `default` (for MinIO or AWS S3) or `md5` (for Cloudflare or Backblaze) +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` ## Log (`log`) @@ -1272,6 +1273,7 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`. - `MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` ## Storage (`storage`) @@ -1286,6 +1288,7 @@ Default storage configuration for attachments, lfs, avatars, repo-avatars, repo- - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` The recommended storage configuration for minio like below: @@ -1307,6 +1310,8 @@ MINIO_USE_SSL = false ; Minio skip SSL verification available when STORAGE_TYPE is `minio` MINIO_INSECURE_SKIP_VERIFY = false SERVE_DIRECT = true +; Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` +MINIO_BUCKET_LOOKUP_TYPE = auto ``` Defaultly every storage has their default base path like below @@ -1353,6 +1358,8 @@ MINIO_LOCATION = us-east-1 MINIO_USE_SSL = false ; Minio skip SSL verification available when STORAGE_TYPE is `minio` MINIO_INSECURE_SKIP_VERIFY = false +; Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` +MINIO_BUCKET_LOOKUP_TYPE = auto ``` ## Repository Archive Storage (`storage.repo-archive`) @@ -1372,6 +1379,7 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`. - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` ## Repository Archives (`repo-archive`) diff --git a/docs/content/administration/config-cheat-sheet.zh-cn.md b/docs/content/administration/config-cheat-sheet.zh-cn.md index 3bb31d3d7..3c6ac8c00 100644 --- a/docs/content/administration/config-cheat-sheet.zh-cn.md +++ b/docs/content/administration/config-cheat-sheet.zh-cn.md @@ -796,6 +796,7 @@ Gitea 创建以下非唯一队列： - `MINIO_USE_SSL`: **false**: Minio 启用 SSL，仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio 跳过 SSL 验证，仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_CHECKSUM_ALGORITHM`: **default**: Minio 校验算法：`default`（适用于 MinIO 或 AWS S3）或 `md5`（适用于 Cloudflare 或 Backblaze） +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio的bucket查找方式默认为`auto`模式，可将其设置为`dns`（虚拟托管样式）或`path`（路径样式），仅当`STORAGE_TYPE`为`minio`时可用。 ## 日志 (`log`) @@ -1201,6 +1202,7 @@ ALLOW_DATA_URI_IMAGES = true - `MINIO_BASE_PATH`：**lfs/**：桶上的 Minio 基本路径，仅在 `STORAGE_TYPE` 为 `minio` 时可用。 - `MINIO_USE_SSL`：**false**：Minio 启用 ssl，仅在 `STORAGE_TYPE` 为 `minio` 时可用。 - `MINIO_INSECURE_SKIP_VERIFY`：**false**：Minio 跳过 SSL 验证，仅在 `STORAGE_TYPE` 为 `minio` 时可用。 +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio的bucket查找方式默认为`auto`模式，可将其设置为`dns`（虚拟托管样式）或`path`（路径样式），仅当`STORAGE_TYPE`为`minio`时可用。 ## 存储 (`storage`) @@ -1215,6 +1217,7 @@ ALLOW_DATA_URI_IMAGES = true - `MINIO_LOCATION`：**us-east-1**：创建桶的 Minio 位置，仅在 `STORAGE_TYPE` 为 `minio` 时可用。 - `MINIO_USE_SSL`：**false**：Minio 启用 ssl，仅在 `STORAGE_TYPE` 为 `minio` 时可用。 - `MINIO_INSECURE_SKIP_VERIFY`：**false**：Minio 跳过 SSL 验证，仅在 `STORAGE_TYPE` 为 `minio` 时可用。 +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio的bucket查找方式默认为`auto`模式，可将其设置为`dns`（虚拟托管样式）或`path`（路径样式），仅当`STORAGE_TYPE`为`minio`时可用。 建议的 minio 存储配置如下： @@ -1236,6 +1239,8 @@ MINIO_USE_SSL = false ; Minio skip SSL verification available when STORAGE_TYPE is `minio` MINIO_INSECURE_SKIP_VERIFY = false SERVE_DIRECT = true +; Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` +MINIO_BUCKET_LOOKUP_TYPE = auto ``` 默认情况下，每个存储都有其默认的基本路径，如下所示： @@ -1282,6 +1287,8 @@ MINIO_LOCATION = us-east-1 MINIO_USE_SSL = false ; Minio skip SSL verification available when STORAGE_TYPE is `minio` MINIO_INSECURE_SKIP_VERIFY = false +; Minio bucket lookup method defaults to auto mode; set it to `dns` for virtual host style or `path` for path style, only available when STORAGE_TYPE is `minio` +MINIO_BUCKET_LOOKUP_TYPE = auto ``` ### 存储库归档存储 (`storage.repo-archive`) @@ -1299,6 +1306,7 @@ MINIO_INSECURE_SKIP_VERIFY = false - `MINIO_BASE_PATH`: **repo-archive/**：存储桶上的Minio基本路径，仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_USE_SSL`: **false**：启用Minio的SSL，仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_INSECURE_SKIP_VERIFY`: **false**：跳过Minio的SSL验证，仅在`STORAGE_TYPE`为`minio`时可用。 +- `MINIO_BUCKET_LOOKUP_TYPE`: **auto**: Minio的bucket查找方式默认为`auto`模式，可将其设置为`dns`（虚拟托管样式）或`path`（路径样式），仅当`STORAGE_TYPE`为`minio`时可用。 ### 存储库归档 (`repo-archive`) diff --git a/modules/setting/storage.go b/modules/setting/storage.go index 0bd52acc0..d80a61a45 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -47,6 +47,7 @@ type MinioStorageConfig struct { InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM" json:",omitempty"` ServeDirect bool `ini:"SERVE_DIRECT"` + BucketLookUpType string `ini:"MINIO_BUCKET_LOOKUP_TYPE" json:",omitempty"` } // Storage represents configuration of storages @@ -82,6 +83,7 @@ func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection { storageSec.Key("MINIO_USE_SSL").MustBool(false) storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default") + storageSec.Key("MINIO_BUCKET_LOOKUP_TYPE").MustString("auto") return storageSec } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index b58ab67dc..986332dfe 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -85,11 +85,23 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) + var lookup minio.BucketLookupType + if config.BucketLookUpType == "auto" || config.BucketLookUpType == "" { + lookup = minio.BucketLookupAuto + } else if config.BucketLookUpType == "dns" { + lookup = minio.BucketLookupDNS + } else if config.BucketLookUpType == "path" { + lookup = minio.BucketLookupPath + } else { + return nil, fmt.Errorf("invalid minio bucket lookup type: %s", config.BucketLookUpType) + } + minioClient, err := minio.New(config.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), - Secure: config.UseSSL, - Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}}, - Region: config.Location, + Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), + Secure: config.UseSSL, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}}, + Region: config.Location, + BucketLookup: lookup, }) if err != nil { return nil, convertMinioErr(err)