Merge pull request 'add bucket lookup type' (#2482) from jilen/forgejo:minio_bucket_lookup into forgejo

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/2482
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
This commit is contained in:
Earl Warren 2024-03-14 09:50:12 +00:00
commit 20e3e671c6
6 changed files with 74 additions and 8 deletions

View file

@ -43,8 +43,11 @@ jobs:
image: 'docker.io/node:20-bookworm' image: 'docker.io/node:20-bookworm'
services: services:
minio: minio:
image: 'docker.io/bitnami/minio:2023.8.31' image: bitnami/minio:2024.2.26
options: >-
--hostname gitea.minio
env: env:
MINIO_DOMAIN: minio
MINIO_ROOT_USER: 123456 MINIO_ROOT_USER: 123456
MINIO_ROOT_PASSWORD: 12345678 MINIO_ROOT_PASSWORD: 12345678
steps: steps:
@ -130,10 +133,10 @@ jobs:
image: 'docker.io/node:20-bookworm' image: 'docker.io/node:20-bookworm'
services: services:
minio: minio:
image: bitnami/minio:2021.3.17 image: bitnami/minio:2024.2.26
env: env:
MINIO_ACCESS_KEY: 123456 MINIO_ROOT_USER: 123456
MINIO_SECRET_KEY: 12345678 MINIO_ROOT_PASSWORD: 12345678
pgsql: pgsql:
image: 'docker.io/postgres:15' image: 'docker.io/postgres:15'
env: env:

View file

@ -842,6 +842,10 @@ Default templates for project boards:
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
- `auto` Auto detect
- `dns` Virtual Host Style bucket lookup
- `path` Path style bucket lookup
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio`
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio` - `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio`
@ -1272,6 +1276,10 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio` - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
- `auto` Auto detect
- `dns` Virtual Host Style bucket lookup
- `path` Path style bucket lookup
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
@ -1287,6 +1295,10 @@ Default storage configuration for attachments, lfs, avatars, repo-avatars, repo-
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio` - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio`
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
- `auto` Auto detect
- `dns` Virtual Host Style bucket lookup
- `path` Path style bucket lookup
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` - `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`
@ -1372,6 +1384,10 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio` - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio bucket lookup type only available when `STORAGE_TYPE` is `minio`
- `auto` Auto detect
- `dns` Virtual Host Style bucket lookup
- `path` Path style bucket lookup
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`

View file

@ -792,6 +792,10 @@ Gitea 创建以下非唯一队列:
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID 以连接,仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_ACCESS_KEY_ID`: Minio accessKeyID 以连接,仅当 STORAGE_TYPE 为 `minio` 时可用。
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey 以连接,仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey 以连接,仅当 STORAGE_TYPE 为 `minio` 时可用。
- `MINIO_BUCKET`: **gitea**: Minio 存储附件的存储桶,仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_BUCKET`: **gitea**: Minio 存储附件的存储桶,仅当 STORAGE_TYPE 为 `minio` 时可用。
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式, 仅当 `STORAGE_TYPE``minio` 时可用。
- `auto` 自动检测
- `dns` 子域名寻址
- `path` 路径寻址
- `MINIO_LOCATION`: **us-east-1**: Minio 存储桶的位置以创建,仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_LOCATION`: **us-east-1**: Minio 存储桶的位置以创建,仅当 STORAGE_TYPE 为 `minio` 时可用。
- `MINIO_BASE_PATH`: **attachments/**: Minio 存储桶上的基本路径,仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_BASE_PATH`: **attachments/**: Minio 存储桶上的基本路径,仅当 STORAGE_TYPE 为 `minio` 时可用。
- `MINIO_USE_SSL`: **false**: Minio 启用 SSL仅当 STORAGE_TYPE 为 `minio` 时可用。 - `MINIO_USE_SSL`: **false**: Minio 启用 SSL仅当 STORAGE_TYPE 为 `minio` 时可用。
@ -1207,6 +1211,10 @@ ALLOW_DATA_URI_IMAGES = true
- `MINIO_ACCESS_KEY_ID`Minio 的 accessKeyID仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_ACCESS_KEY_ID`Minio 的 accessKeyID仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_SECRET_ACCESS_KEY`Minio 的 secretAccessKey仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_SECRET_ACCESS_KEY`Minio 的 secretAccessKey仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_BUCKET`**gitea**:用于存储 lfs 的 Minio 桶,仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_BUCKET`**gitea**:用于存储 lfs 的 Minio 桶,仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式,可选值为 `auto`, `dns`, `path` 仅当 `STORAGE_TYPE``minio` 时可用。
- `auto` 自动检测
- `dns` 子域名寻址
- `path` 路径寻址
- `MINIO_LOCATION`**us-east-1**:创建桶的 Minio 位置,仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_LOCATION`**us-east-1**:创建桶的 Minio 位置,仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_BASE_PATH`**lfs/**:桶上的 Minio 基本路径,仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_BASE_PATH`**lfs/**:桶上的 Minio 基本路径,仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_USE_SSL`**false**Minio 启用 ssl仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_USE_SSL`**false**Minio 启用 ssl仅在 `STORAGE_TYPE``minio` 时可用。
@ -1222,6 +1230,10 @@ ALLOW_DATA_URI_IMAGES = true
- `MINIO_ACCESS_KEY_ID`Minio 的 accessKeyID仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_ACCESS_KEY_ID`Minio 的 accessKeyID仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_SECRET_ACCESS_KEY`Minio 的 secretAccessKey仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_SECRET_ACCESS_KEY`Minio 的 secretAccessKey仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_BUCKET`**gitea**:用于存储数据的 Minio 桶,仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_BUCKET`**gitea**:用于存储数据的 Minio 桶,仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式,可选值为 `auto`, `dns`, `path` 仅当 `STORAGE_TYPE``minio` 时可用。
- `auto` 自动检测
- `dns` 子域名寻址
- `path` 路径寻址
- `MINIO_LOCATION`**us-east-1**:创建桶的 Minio 位置,仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_LOCATION`**us-east-1**:创建桶的 Minio 位置,仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_USE_SSL`**false**Minio 启用 ssl仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_USE_SSL`**false**Minio 启用 ssl仅在 `STORAGE_TYPE``minio` 时可用。
- `MINIO_INSECURE_SKIP_VERIFY`**false**Minio 跳过 SSL 验证,仅在 `STORAGE_TYPE``minio` 时可用。 - `MINIO_INSECURE_SKIP_VERIFY`**false**Minio 跳过 SSL 验证,仅在 `STORAGE_TYPE``minio` 时可用。
@ -1305,6 +1317,10 @@ MINIO_INSECURE_SKIP_VERIFY = false
- `MINIO_ACCESS_KEY_ID`: Minio的accessKeyID仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_ACCESS_KEY_ID`: Minio的accessKeyID仅在`STORAGE_TYPE`为`minio`时可用。
- `MINIO_SECRET_ACCESS_KEY`: Minio的secretAccessKey仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_SECRET_ACCESS_KEY`: Minio的secretAccessKey仅在`STORAGE_TYPE`为`minio`时可用。
- `MINIO_BUCKET`: **gitea**用于存储归档的Minio存储桶仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_BUCKET`: **gitea**用于存储归档的Minio存储桶仅在`STORAGE_TYPE`为`minio`时可用。
- `MINIO_BUCKET_LOOKUP`: **auto**: Minio 存储桶寻址方式,可选值为 `auto`, `dns`, `path` 仅当 `STORAGE_TYPE``minio` 时可用。
- `auto` 自动检测
- `dns` 子域名寻址
- `path` 路径寻址
- `MINIO_LOCATION`: **us-east-1**用于创建存储桶的Minio位置仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_LOCATION`: **us-east-1**用于创建存储桶的Minio位置仅在`STORAGE_TYPE`为`minio`时可用。
- `MINIO_BASE_PATH`: **repo-archive/**存储桶上的Minio基本路径仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_BASE_PATH`: **repo-archive/**存储桶上的Minio基本路径仅在`STORAGE_TYPE`为`minio`时可用。
- `MINIO_USE_SSL`: **false**启用Minio的SSL仅在`STORAGE_TYPE`为`minio`时可用。 - `MINIO_USE_SSL`: **false**启用Minio的SSL仅在`STORAGE_TYPE`为`minio`时可用。

View file

@ -41,6 +41,7 @@ type MinioStorageConfig struct {
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"` AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"` SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"`
Bucket string `ini:"MINIO_BUCKET" json:",omitempty"` Bucket string `ini:"MINIO_BUCKET" json:",omitempty"`
BucketLookup string `ini:"MINIO_BUCKET_LOOKUP" json:",omitempty"`
Location string `ini:"MINIO_LOCATION" json:",omitempty"` Location string `ini:"MINIO_LOCATION" json:",omitempty"`
BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"` BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"`
UseSSL bool `ini:"MINIO_USE_SSL"` UseSSL bool `ini:"MINIO_USE_SSL"`
@ -78,6 +79,7 @@ func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("") storageSec.Key("MINIO_ACCESS_KEY_ID").MustString("")
storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") storageSec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
storageSec.Key("MINIO_BUCKET").MustString("gitea") storageSec.Key("MINIO_BUCKET").MustString("gitea")
storageSec.Key("MINIO_BUCKET_LOOKUP").MustString("auto")
storageSec.Key("MINIO_LOCATION").MustString("us-east-1") storageSec.Key("MINIO_LOCATION").MustString("us-east-1")
storageSec.Key("MINIO_USE_SSL").MustBool(false) storageSec.Key("MINIO_USE_SSL").MustBool(false)
storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)

View file

@ -82,6 +82,17 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" { if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm) return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
} }
var lookup minio.BucketLookupType
switch config.BucketLookup {
case "auto", "":
lookup = minio.BucketLookupAuto
case "dns":
lookup = minio.BucketLookupDNS
case "path":
lookup = minio.BucketLookupPath
default:
return nil, fmt.Errorf("invalid minio bucket lookup type %s", config.BucketLookup)
}
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
@ -90,6 +101,7 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
Secure: config.UseSSL, Secure: config.UseSSL,
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}}, Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
Region: config.Location, Region: config.Location,
BucketLookup: lookup,
}) })
if err != nil { if err != nil {
return nil, convertMinioErr(err) return nil, convertMinioErr(err)

View file

@ -31,6 +31,23 @@ func TestMinioStorageIterator(t *testing.T) {
}) })
} }
func TestVirtualHostMinioStorage(t *testing.T) {
if os.Getenv("CI") == "" {
t.Skip("minioStorage not present outside of CI")
return
}
testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
MinioConfig: setting.MinioStorageConfig{
Endpoint: "minio:9000",
AccessKeyID: "123456",
SecretAccessKey: "12345678",
Bucket: "gitea",
Location: "us-east-1",
BucketLookup: "dns",
},
})
}
func TestMinioStoragePath(t *testing.T) { func TestMinioStoragePath(t *testing.T) {
m := &MinioStorage{basePath: ""} m := &MinioStorage{basePath: ""}
assert.Equal(t, "", m.buildMinioPath("/")) assert.Equal(t, "", m.buildMinioPath("/"))