forgejo/models/db/engine.go

302 lines
8.7 KiB
Go
Raw Normal View History

2014-02-12 17:49:46 +00:00
// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
2014-02-12 17:49:46 +00:00
package db
2014-02-18 22:48:02 +00:00
import (
"context"
2014-10-19 05:35:24 +00:00
"database/sql"
2014-02-18 22:48:02 +00:00
"fmt"
"io"
"reflect"
"strings"
2014-02-18 22:48:02 +00:00
"code.gitea.io/gitea/modules/setting"
2019-10-17 09:26:49 +00:00
"xorm.io/xorm"
"xorm.io/xorm/names"
"xorm.io/xorm/schemas"
2016-11-26 00:20:18 +00:00
_ "github.com/denisenkom/go-mssqldb" // Needed for the MSSQL driver
_ "github.com/go-sql-driver/mysql" // Needed for the MySQL driver
_ "github.com/lib/pq" // Needed for the Postgresql driver
2014-02-18 22:48:02 +00:00
)
var (
x *xorm.Engine
tables []any
initFuncs []func() error
)
2014-10-19 05:35:24 +00:00
// Engine represents a xorm engine or session.
type Engine interface {
Table(tableNameOrBean any) *xorm.Session
Count(...any) (int64, error)
Decr(column string, arg ...any) *xorm.Session
Delete(...any) (int64, error)
Truncate(...any) (int64, error)
Exec(...any) (sql.Result, error)
Find(any, ...any) error
Get(beans ...any) (bool, error)
ID(any) *xorm.Session
In(string, ...any) *xorm.Session
Incr(column string, arg ...any) *xorm.Session
Insert(...any) (int64, error)
Iterate(any, xorm.IterFunc) error
[SEMVER] store SemVer in ForgejoSemVer after a database upgrade (cherry picked from commit b7fe7cf401f4bddd6455efc651f7ac054f3fe1cf) (cherry picked from commit cf339eed4f4851b18448dbdd83df32d00bc6f45b) (cherry picked from commit 4f3a16168bbeced519a60c32e10e2895d9367238) (cherry picked from commit 6f5bbc53fcebd614f5ee8627c7d6e3c637ffa694) (cherry picked from commit aca42b422e76668387769c15868ef77b073bb7db) (cherry picked from commit 5a7f7580e525694d1f27e12329c8532bd29273f8) (cherry picked from commit 06c383c807ad49e1e35b429a10c6a11c65aeebe5) (cherry picked from commit fe831dcb53b81b6cc632be751bfcbfb9bc00efd3) (cherry picked from commit cd12cd0dbce47c6117ea579ae5019c182155b3a9) (cherry picked from commit cc79163703ce31706c86b88c38fb8a20ed745e20) (cherry picked from commit 0102a5715ea6a03b560cc2f0b6cbe2b2576c255e) (cherry picked from commit 403f7520b3056eace36eae505afbab6a05f597b7) (cherry picked from commit a3b61510a246f61c174ddc3c288e556522a6aab9) (cherry picked from commit f83f0f9feb76a8b62ca4d74dc2785c713fbec282) (cherry picked from commit fd1c3a6d09a057070844cd955d0920518e60f408) (cherry picked from commit f7cdc3d6f1ad1ff01bb1814207fcb41210db80e8) (cherry picked from commit 060121b644e0515a6b673cdd514d52e6fa1e6ec3) (cherry picked from commit 62c847ff0235196e73e860d0d1658f0734985270) (cherry picked from commit 4d051b51c2813828b978ef84b6b72c0ba051741b) (cherry picked from commit 86e6981a936b1937065605892083b2488424bf5e) (cherry picked from commit c1fc9e441b1467033911f3848f82a89aaacf98a9) (cherry picked from commit 8bb2f0871a507d881acf74acf8fb90fd5ebff567) (cherry picked from commit 0cd9fe52511ee06f1b8849325d196f60d8ccd151) (cherry picked from commit b0b44778b4d3e51852d0669dfe6d9a6c334b90e4) (cherry picked from commit 7c2f4f749f1c8986875eebf95254a3db151e5248)
2023-08-08 21:55:25 +00:00
IsTableExist(any) (bool, error)
Join(joinOperator string, tablename, condition any, args ...any) *xorm.Session
SQL(any, ...any) *xorm.Session
Where(any, ...any) *xorm.Session
2019-06-12 19:41:28 +00:00
Asc(colNames ...string) *xorm.Session
Desc(colNames ...string) *xorm.Session
API add/generalize pagination (#9452) * paginate results * fixed deadlock * prevented breaking change * updated swagger * go fmt * fixed find topic * go mod tidy * go mod vendor with go1.13.5 * fixed repo find topics * fixed unit test * added Limit method to Engine struct; use engine variable when provided; fixed gitignore * use ItemsPerPage for default pagesize; fix GetWatchers, getOrgUsersByOrgID and GetStargazers; fix GetAllCommits headers; reverted some changed behaviors * set Page value on Home route * improved memory allocations * fixed response headers * removed logfiles * fixed import order * import order * improved swagger * added function to get models.ListOptions from context * removed pagesize diff on unit test * fixed imports * removed unnecessary struct field * fixed go fmt * scoped PR * code improvements * code improvements * go mod tidy * fixed import order * fixed commit statuses session * fixed files headers * fixed headers; added pagination for notifications * go mod tidy * go fmt * removed Private from user search options; added setting.UI.IssuePagingNum as default valeu on repo's issues list * Apply suggestions from code review Co-Authored-By: 6543 <6543@obermui.de> Co-Authored-By: zeripath <art27@cantab.net> * fixed build error * CI.restart() * fixed merge conflicts resolve * fixed conflicts resolve * improved FindTrackedTimesOptions.ToOptions() method * added backwards compatibility on ListReleases request; fixed issue tracked time ToSession * fixed build error; fixed swagger template * fixed swagger template * fixed ListReleases backwards compatibility * added page to user search route Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath <art27@cantab.net>
2020-01-24 19:00:29 +00:00
Limit(limit int, start ...int) *xorm.Session
NoAutoTime() *xorm.Session
SumInt(bean any, columnName string) (res int64, err error)
Sync(...any) error
Select(string) *xorm.Session
NotIn(string, ...any) *xorm.Session
OrderBy(any, ...any) *xorm.Session
Exist(...any) (bool, error)
Distinct(...string) *xorm.Session
Query(...any) ([]map[string][]byte, error)
Cols(...string) *xorm.Session
Context(ctx context.Context) *xorm.Session
Ping() error
2014-10-19 05:35:24 +00:00
}
// TableInfo returns table's information via an object
func TableInfo(v any) (*schemas.Table, error) {
return x.TableInfo(v)
}
// DumpTables dump tables information
func DumpTables(tables []*schemas.Table, w io.Writer, tp ...schemas.DBType) error {
return x.DumpTables(tables, w, tp...)
}
2016-11-26 00:20:18 +00:00
// RegisterModel registers model, if initfunc provided, it will be invoked after data model sync
func RegisterModel(bean any, initFunc ...func() error) {
tables = append(tables, bean)
if len(initFuncs) > 0 && initFunc[0] != nil {
initFuncs = append(initFuncs, initFunc[0])
}
}
2014-03-21 05:48:10 +00:00
2014-04-05 14:46:32 +00:00
func init() {
2016-11-26 00:20:18 +00:00
gonicNames := []string{"SSL", "UID"}
2015-08-27 15:06:14 +00:00
for _, name := range gonicNames {
names.LintGonicMapper[name] = true
2015-08-27 15:06:14 +00:00
}
2014-04-05 14:46:32 +00:00
}
// newXORMEngine returns a new XORM engine from the configuration
func newXORMEngine() (*xorm.Engine, error) {
connStr, err := setting.DBConnStr()
if err != nil {
return nil, err
2014-03-30 14:47:08 +00:00
}
var engine *xorm.Engine
if setting.Database.Type.IsPostgreSQL() && len(setting.Database.Schema) > 0 {
// OK whilst we sort out our schema issues - create a schema aware postgres
registerPostgresSchemaDriver()
engine, err = xorm.NewEngine("postgresschema", connStr)
} else {
engine, err = xorm.NewEngine(setting.Database.Type.String(), connStr)
}
if err != nil {
return nil, err
}
if setting.Database.Type == "mysql" {
engine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"})
} else if setting.Database.Type == "mssql" {
engine.Dialect().SetParams(map[string]string{"DEFAULT_VARCHAR": "nvarchar"})
}
engine.SetSchema(setting.Database.Schema)
return engine, nil
}
// SyncAllTables sync the schemas of all tables, is required by unit test code
func SyncAllTables() error {
Fix wrong warn messages in migration steps (#25475) The recent change on xorm for `Sync` is it will not warn when database have columns which is not listed on struct. So we just need this warn logs when `Sync` the whole database but not in the migrations Sync. This PR will remove almost unnecessary warning logs on migrations. Now below logs in CI will disappear. ```log 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column creator_id but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column is_closed but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column board_type but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column type but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column closed_date_unix but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column created_unix but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column updated_unix but struct has not related field 2023/06/23 17:51:32 models/db/engine.go:191:InitEngineWithMigration() [W] Table gtestschema.project has column card_type but struct has not related field ```
2023-06-24 08:20:08 +00:00
_, err := x.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{
WarnIfDatabaseColumnMissed: true,
}, tables...)
return err
}
// InitEngine initializes the xorm.Engine and sets it as db.DefaultContext
func InitEngine(ctx context.Context) error {
xormEngine, err := newXORMEngine()
2014-02-18 22:48:02 +00:00
if err != nil {
return fmt.Errorf("failed to connect to database: %w", err)
2014-02-18 22:48:02 +00:00
}
xormEngine.SetMapper(names.GonicMapper{})
2014-12-07 01:22:48 +00:00
// WARNING: for serv command, MUST remove the output to os.stdout,
2014-03-20 20:04:56 +00:00
// so use log file to instead print to stdout.
xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL))
xormEngine.ShowSQL(setting.Database.LogSQL)
xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns)
xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns)
xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
xormEngine.SetDefaultContext(ctx)
SetDefaultEngine(ctx, xormEngine)
return nil
}
// SetDefaultEngine sets the default engine for db
func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) {
x = eng
DefaultContext = &Context{
Context: ctx,
e: x,
}
2014-02-18 22:48:02 +00:00
}
// UnsetDefaultEngine closes and unsets the default engine
// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now,
// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `x` and DefaultContext without close
// Global database engine related functions are all racy and there is no graceful close right now.
func UnsetDefaultEngine() {
if x != nil {
_ = x.Close()
x = nil
}
DefaultContext = nil
}
// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext
// This function must never call .Sync() if the provided migration function fails.
// When called from the "doctor" command, the migration function is a version check
// that prevents the doctor from fixing anything in the database if the migration level
// is different from the expected value.
func InitEngineWithMigration(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
if err = InitEngine(ctx); err != nil {
return err
2014-04-05 14:46:32 +00:00
}
2015-01-22 12:49:52 +00:00
if err = x.Ping(); err != nil {
return err
}
preprocessDatabaseCollation(x)
// We have to run migrateFunc here in case the user is re-running installation on a previously created DB.
// If we do not then table schemas will be changed and there will be conflicts when the migrations run properly.
//
// Installation should only be being re-run if users want to recover an old database.
// However, we should think carefully about should we support re-install on an installed instance,
// as there may be other problems due to secret reinitialization.
if err = migrateFunc(x); err != nil {
return fmt.Errorf("migrate: %w", err)
2015-01-22 12:49:52 +00:00
}
if err = SyncAllTables(); err != nil {
return fmt.Errorf("sync database struct error: %w", err)
2014-02-19 09:50:53 +00:00
}
2015-01-23 07:54:16 +00:00
for _, initFunc := range initFuncs {
if err := initFunc(); err != nil {
return fmt.Errorf("initFunc failed: %w", err)
}
}
return nil
2014-02-18 22:48:02 +00:00
}
2014-03-20 20:04:56 +00:00
// NamesToBean return a list of beans or an error
func NamesToBean(names ...string) ([]any, error) {
beans := []any{}
if len(names) == 0 {
beans = append(beans, tables...)
return beans, nil
}
// Need to map provided names to beans...
beanMap := make(map[string]any)
for _, bean := range tables {
beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean
beanMap[strings.ToLower(x.TableName(bean))] = bean
beanMap[strings.ToLower(x.TableName(bean, true))] = bean
}
gotBean := make(map[any]bool)
for _, name := range names {
bean, ok := beanMap[strings.ToLower(strings.TrimSpace(name))]
if !ok {
return nil, fmt.Errorf("no table found that matches: %s", name)
}
if !gotBean[bean] {
beans = append(beans, bean)
gotBean[bean] = true
}
}
return beans, nil
}
// DumpDatabase dumps all data from database according the special database SQL syntax to file system.
func DumpDatabase(filePath, dbType string) error {
var tbs []*schemas.Table
for _, t := range tables {
t, err := x.TableInfo(t)
if err != nil {
return err
}
tbs = append(tbs, t)
}
type Version struct {
ID int64 `xorm:"pk autoincr"`
Version int64
}
t, err := x.TableInfo(&Version{})
if err != nil {
return err
}
tbs = append(tbs, t)
if len(dbType) > 0 {
return x.DumpTablesToFile(tbs, filePath, schemas.DBType(dbType))
}
return x.DumpTablesToFile(tbs, filePath)
2014-05-05 04:55:17 +00:00
}
// MaxBatchInsertSize returns the table's max batch insert size
func MaxBatchInsertSize(bean any) int {
t, err := x.TableInfo(bean)
if err != nil {
return 50
}
return 999 / len(t.ColumnsSeq())
}
// IsTableNotEmpty returns true if table has at least one record
func IsTableNotEmpty(tableName string) (bool, error) {
return x.Table(tableName).Exist()
}
// DeleteAllRecords will delete all the records of this table
func DeleteAllRecords(tableName string) error {
_, err := x.Exec(fmt.Sprintf("DELETE FROM %s", tableName))
return err
}
// GetMaxID will return max id of the table
func GetMaxID(beanOrTableName any) (maxID int64, err error) {
_, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID)
return maxID, err
}
func SetLogSQL(ctx context.Context, on bool) {
e := GetEngine(ctx)
if x, ok := e.(*xorm.Engine); ok {
x.ShowSQL(on)
} else if sess, ok := e.(*xorm.Session); ok {
sess.Engine().ShowSQL(on)
}
}