diff --git a/CHANGELOG.md b/CHANGELOG.md index e6be4719..fdae39ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Reduced memory consumption (you can expect ~20% less allocated memory). -- Added support for split (async and sync) DB connections pool increasing even further the concurrent throughput. +- Added support for split (concurrent and nonconcurrent) DB connections pool increasing even further the concurrent throughput without blocking reads on heavy write load. - Improved record references delete performance. diff --git a/core/app.go b/core/app.go index c6e02878..e1bc76bb 100644 --- a/core/app.go +++ b/core/app.go @@ -19,7 +19,7 @@ type App interface { // Deprecated: // This method may get removed in the near future. // It is recommended to access the logs db instance from app.Dao().DB() or - // if you want more flexibility - app.Dao().AsyncDB() and app.Dao().SyncDB(). + // if you want more flexibility - app.Dao().ConcurrentDB() and app.Dao().NonconcurrentDB(). // // DB returns the default app database instance. DB() *dbx.DB @@ -34,7 +34,7 @@ type App interface { // Deprecated: // This method may get removed in the near future. // It is recommended to access the logs db instance from app.LogsDao().DB() or - // if you want more flexibility - app.LogsDao().AsyncDB() and app.LogsDao().SyncDB(). + // if you want more flexibility - app.LogsDao().ConcurrentDB() and app.LogsDao().NonconcurrentDB(). // // LogsDB returns the app logs database instance. LogsDB() *dbx.DB diff --git a/core/base.go b/core/base.go index 05ae117f..28447d6b 100644 --- a/core/base.go +++ b/core/base.go @@ -316,19 +316,19 @@ func (app *BaseApp) Bootstrap() error { // (eg. closing db connections). func (app *BaseApp) ResetBootstrapState() error { if app.Dao() != nil { - if err := app.Dao().AsyncDB().(*dbx.DB).Close(); err != nil { + if err := app.Dao().ConcurrentDB().(*dbx.DB).Close(); err != nil { return err } - if err := app.Dao().SyncDB().(*dbx.DB).Close(); err != nil { + if err := app.Dao().NonconcurrentDB().(*dbx.DB).Close(); err != nil { return err } } if app.LogsDao() != nil { - if err := app.LogsDao().AsyncDB().(*dbx.DB).Close(); err != nil { + if err := app.LogsDao().ConcurrentDB().(*dbx.DB).Close(); err != nil { return err } - if err := app.LogsDao().SyncDB().(*dbx.DB).Close(); err != nil { + if err := app.LogsDao().NonconcurrentDB().(*dbx.DB).Close(); err != nil { return err } } @@ -343,7 +343,7 @@ func (app *BaseApp) ResetBootstrapState() error { // Deprecated: // This method may get removed in the near future. // It is recommended to access the db instance from app.Dao().DB() or -// if you want more flexibility - app.Dao().AsyncDB() and app.Dao().SyncDB(). +// if you want more flexibility - app.Dao().ConcurrentDB() and app.Dao().NonconcurrentDB(). // // DB returns the default app database instance. func (app *BaseApp) DB() *dbx.DB { @@ -367,7 +367,7 @@ func (app *BaseApp) Dao() *daos.Dao { // Deprecated: // This method may get removed in the near future. // It is recommended to access the logs db instance from app.LogsDao().DB() or -// if you want more flexibility - app.LogsDao().AsyncDB() and app.LogsDao().SyncDB(). +// if you want more flexibility - app.LogsDao().ConcurrentDB() and app.LogsDao().NonconcurrentDB(). // // LogsDB returns the app logs database instance. func (app *BaseApp) LogsDB() *dbx.DB { @@ -826,23 +826,23 @@ func (app *BaseApp) initLogsDB() error { maxIdleConns = app.logsMaxIdleConns } - asyncDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db")) + concurrentDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db")) if err != nil { return err } - asyncDB.DB().SetMaxOpenConns(maxOpenConns) - asyncDB.DB().SetMaxIdleConns(maxIdleConns) - asyncDB.DB().SetConnMaxIdleTime(5 * time.Minute) + concurrentDB.DB().SetMaxOpenConns(maxOpenConns) + concurrentDB.DB().SetMaxIdleConns(maxIdleConns) + concurrentDB.DB().SetConnMaxIdleTime(5 * time.Minute) - syncDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db")) + nonconcurrentDB, err := connectDB(filepath.Join(app.DataDir(), "logs.db")) if err != nil { return err } - syncDB.DB().SetMaxOpenConns(1) - syncDB.DB().SetMaxIdleConns(1) - syncDB.DB().SetConnMaxIdleTime(5 * time.Minute) + nonconcurrentDB.DB().SetMaxOpenConns(1) + nonconcurrentDB.DB().SetMaxIdleConns(1) + nonconcurrentDB.DB().SetConnMaxIdleTime(5 * time.Minute) - app.logsDao = daos.NewMultiDB(asyncDB, syncDB) + app.logsDao = daos.NewMultiDB(concurrentDB, nonconcurrentDB) return nil } @@ -857,41 +857,41 @@ func (app *BaseApp) initDataDB() error { maxIdleConns = app.dataMaxIdleConns } - asyncDB, err := connectDB(filepath.Join(app.DataDir(), "data.db")) + concurrentDB, err := connectDB(filepath.Join(app.DataDir(), "data.db")) if err != nil { return err } - asyncDB.DB().SetMaxOpenConns(maxOpenConns) - asyncDB.DB().SetMaxIdleConns(maxIdleConns) - asyncDB.DB().SetConnMaxIdleTime(5 * time.Minute) + concurrentDB.DB().SetMaxOpenConns(maxOpenConns) + concurrentDB.DB().SetMaxIdleConns(maxIdleConns) + concurrentDB.DB().SetConnMaxIdleTime(5 * time.Minute) - syncDB, err := connectDB(filepath.Join(app.DataDir(), "data.db")) + nonconcurrentDB, err := connectDB(filepath.Join(app.DataDir(), "data.db")) if err != nil { return err } - syncDB.DB().SetMaxOpenConns(1) - syncDB.DB().SetMaxIdleConns(1) - syncDB.DB().SetConnMaxIdleTime(5 * time.Minute) + nonconcurrentDB.DB().SetMaxOpenConns(1) + nonconcurrentDB.DB().SetMaxIdleConns(1) + nonconcurrentDB.DB().SetConnMaxIdleTime(5 * time.Minute) if app.IsDebug() { - syncDB.QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) { + nonconcurrentDB.QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) { color.HiBlack("[%.2fms] %v\n", float64(t.Milliseconds()), sql) } - asyncDB.QueryLogFunc = syncDB.QueryLogFunc + concurrentDB.QueryLogFunc = nonconcurrentDB.QueryLogFunc - syncDB.ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) { + nonconcurrentDB.ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) { color.HiBlack("[%.2fms] %v\n", float64(t.Milliseconds()), sql) } - asyncDB.ExecLogFunc = syncDB.ExecLogFunc + concurrentDB.ExecLogFunc = nonconcurrentDB.ExecLogFunc } - app.dao = app.createDaoWithHooks(asyncDB, syncDB) + app.dao = app.createDaoWithHooks(concurrentDB, nonconcurrentDB) return nil } -func (app *BaseApp) createDaoWithHooks(asyncDB, syncDB dbx.Builder) *daos.Dao { - dao := daos.NewMultiDB(asyncDB, syncDB) +func (app *BaseApp) createDaoWithHooks(concurrentDB, nonconcurrentDB dbx.Builder) *daos.Dao { + dao := daos.NewMultiDB(concurrentDB, nonconcurrentDB) dao.BeforeCreateFunc = func(eventDao *daos.Dao, m models.Model) error { return app.OnModelBeforeCreate().Trigger(&ModelEvent{eventDao, m}) diff --git a/core/base_test.go b/core/base_test.go index ab5664dc..e23b6aa9 100644 --- a/core/base_test.go +++ b/core/base_test.go @@ -143,16 +143,16 @@ func TestBaseAppGetters(t *testing.T) { t.Fatalf("Expected app.Dao %v, got %v", app.Dao(), app.dao) } - if app.dao.AsyncDB() != app.DB() { - t.Fatalf("Expected app.DB %v, got %v", app.DB(), app.dao.AsyncDB()) + if app.dao.ConcurrentDB() != app.DB() { + t.Fatalf("Expected app.DB %v, got %v", app.DB(), app.dao.ConcurrentDB()) } if app.logsDao != app.LogsDao() { t.Fatalf("Expected app.LogsDao %v, got %v", app.LogsDao(), app.logsDao) } - if app.logsDao.AsyncDB() != app.LogsDB() { - t.Fatalf("Expected app.LogsDB %v, got %v", app.LogsDB(), app.logsDao.AsyncDB()) + if app.logsDao.ConcurrentDB() != app.LogsDB() { + t.Fatalf("Expected app.LogsDB %v, got %v", app.LogsDB(), app.logsDao.ConcurrentDB()) } if app.dataDir != app.DataDir() { diff --git a/daos/base.go b/daos/base.go index b78f4e70..9c26dda2 100644 --- a/daos/base.go +++ b/daos/base.go @@ -25,10 +25,10 @@ func New(db dbx.Builder) *Dao { // New creates a new Dao instance with the provided dedicated // async and sync db builders. -func NewMultiDB(asyncDB, syncDB dbx.Builder) *Dao { +func NewMultiDB(concurrentDB, nonconcurrentDB dbx.Builder) *Dao { return &Dao{ - asyncDB: asyncDB, - syncDB: syncDB, + concurrentDB: concurrentDB, + nonconcurrentDB: nonconcurrentDB, } } @@ -36,8 +36,8 @@ func NewMultiDB(asyncDB, syncDB dbx.Builder) *Dao { // Think of Dao as a repository and service layer in one. type Dao struct { // in a transaction both refer to the same *dbx.TX instance - asyncDB dbx.Builder - syncDB dbx.Builder + concurrentDB dbx.Builder + nonconcurrentDB dbx.Builder // @todo delete after removing Block and Continue sem *semaphore.Weighted @@ -53,26 +53,28 @@ type Dao struct { // DB returns the default dao db builder (*dbx.DB or *dbx.TX). // -// Currently the default db builder is dao.asyncDB but that may change in the future. +// Currently the default db builder is dao.concurrentDB but that may change in the future. func (dao *Dao) DB() dbx.Builder { - return dao.AsyncDB() + return dao.ConcurrentDB() } -// AsyncDB returns the dao asynchronous db builder (*dbx.DB or *dbx.TX). +// ConcurrentDB returns the dao concurrent (aka. multiple open connections) +// db builder (*dbx.DB or *dbx.TX). // -// In a transaction the asyncDB and syncDB refer to the same *dbx.TX instance. -func (dao *Dao) AsyncDB() dbx.Builder { - return dao.asyncDB +// In a transaction the concurrentDB and nonconcurrentDB refer to the same *dbx.TX instance. +func (dao *Dao) ConcurrentDB() dbx.Builder { + return dao.concurrentDB } -// SyncDB returns the dao synchronous db builder (*dbx.DB or *dbx.TX). +// NonconcurrentDB returns the dao nonconcurrent (aka. single open connection) +// db builder (*dbx.DB or *dbx.TX). // -// In a transaction the asyncDB and syncDB refer to the same *dbx.TX instance. -func (dao *Dao) SyncDB() dbx.Builder { - return dao.syncDB +// In a transaction the concurrentDB and nonconcurrentDB refer to the same *dbx.TX instance. +func (dao *Dao) NonconcurrentDB() dbx.Builder { + return dao.nonconcurrentDB } -// Deprecated: Will be removed in the next releases. Use [Dao.SyncDB()] instead. +// Deprecated: Will be removed in the next releases. Use [Dao.NonconcurrentDB()] instead. // // Block acquires a lock and blocks all other go routines that uses // the Dao instance until dao.Continue() is called, effectively making @@ -105,7 +107,7 @@ func (dao *Dao) Block(ctx context.Context) error { return dao.sem.Acquire(ctx, 1) } -// Deprecated: Will be removed in the next releases. Use [Dao.SyncDB()] instead. +// Deprecated: Will be removed in the next releases. Use [Dao.NonconcurrentDB()] instead. // // Continue releases the previously acquired Block() lock. func (dao *Dao) Continue() { @@ -139,7 +141,7 @@ type afterCallGroup struct { // // It is safe to nest RunInTransaction calls as long as you use the txDao. func (dao *Dao) RunInTransaction(fn func(txDao *Dao) error) error { - switch txOrDB := dao.SyncDB().(type) { + switch txOrDB := dao.NonconcurrentDB().(type) { case *dbx.Tx: // nested transactions are not supported by default // so execute the function within the current transaction @@ -213,7 +215,7 @@ func (dao *Dao) RunInTransaction(fn func(txDao *Dao) error) error { return txError } - return errors.New("Failed to start transaction (unknown dao.db)") + return errors.New("failed to start transaction (unknown dao.NonconcurrentDB() instance)") } // Delete deletes the provided model. @@ -229,7 +231,7 @@ func (dao *Dao) Delete(m models.Model) error { } } - if err := retryDao.SyncDB().Model(m).Delete(); err != nil { + if err := retryDao.NonconcurrentDB().Model(m).Delete(); err != nil { return err } @@ -274,7 +276,7 @@ func (dao *Dao) update(m models.Model) error { if v, ok := any(m).(models.ColumnValueMapper); ok { dataMap := v.ColumnValueMap() - _, err := dao.SyncDB().Update( + _, err := dao.NonconcurrentDB().Update( m.TableName(), dataMap, dbx.HashExp{"id": m.GetId()}, @@ -284,7 +286,7 @@ func (dao *Dao) update(m models.Model) error { return err } } else { - if err := dao.SyncDB().Model(m).Update(); err != nil { + if err := dao.NonconcurrentDB().Model(m).Update(); err != nil { return err } } @@ -325,12 +327,12 @@ func (dao *Dao) create(m models.Model) error { dataMap["id"] = m.GetId() } - _, err := dao.SyncDB().Insert(m.TableName(), dataMap).Execute() + _, err := dao.NonconcurrentDB().Insert(m.TableName(), dataMap).Execute() if err != nil { return err } } else { - if err := dao.SyncDB().Model(m).Insert(); err != nil { + if err := dao.NonconcurrentDB().Model(m).Insert(); err != nil { return err } } @@ -353,7 +355,7 @@ Retry: if attempts == 2 { // assign new Dao without the before hooks to avoid triggering // the already fired before callbacks multiple times - retryDao = NewMultiDB(dao.asyncDB, dao.syncDB) + retryDao = NewMultiDB(dao.concurrentDB, dao.nonconcurrentDB) retryDao.AfterCreateFunc = dao.AfterCreateFunc retryDao.AfterUpdateFunc = dao.AfterUpdateFunc retryDao.AfterDeleteFunc = dao.AfterDeleteFunc diff --git a/daos/base_test.go b/daos/base_test.go index 84da6716..af260d3c 100644 --- a/daos/base_test.go +++ b/daos/base_test.go @@ -24,17 +24,17 @@ func TestNewMultiDB(t *testing.T) { testApp, _ := tests.NewTestApp() defer testApp.Cleanup() - dao := daos.NewMultiDB(testApp.Dao().AsyncDB(), testApp.Dao().SyncDB()) + dao := daos.NewMultiDB(testApp.Dao().ConcurrentDB(), testApp.Dao().NonconcurrentDB()) - if dao.DB() != testApp.Dao().AsyncDB() { + if dao.DB() != testApp.Dao().ConcurrentDB() { t.Fatal("[db-asyncdb] The 2 db instances are different") } - if dao.AsyncDB() != testApp.Dao().AsyncDB() { + if dao.ConcurrentDB() != testApp.Dao().ConcurrentDB() { t.Fatal("[asyncdb-asyncdb] The 2 db instances are different") } - if dao.SyncDB() != testApp.Dao().SyncDB() { + if dao.NonconcurrentDB() != testApp.Dao().NonconcurrentDB() { t.Fatal("[syncdb-syncdb] The 2 db instances are different") } } diff --git a/daos/record_test.go b/daos/record_test.go index d4208b4e..bb72ed9a 100644 --- a/daos/record_test.go +++ b/daos/record_test.go @@ -634,16 +634,16 @@ func TestDeleteRecord(t *testing.T) { // delete existing record + cascade // --- calledQueries := []string{} - app.Dao().SyncDB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) { + app.Dao().NonconcurrentDB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) { calledQueries = append(calledQueries, sql) } - app.Dao().AsyncDB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) { + app.Dao().ConcurrentDB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) { calledQueries = append(calledQueries, sql) } - app.Dao().SyncDB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) { + app.Dao().NonconcurrentDB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) { calledQueries = append(calledQueries, sql) } - app.Dao().AsyncDB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) { + app.Dao().ConcurrentDB().(*dbx.DB).ExecLogFunc = func(ctx context.Context, t time.Duration, sql string, result sql.Result, err error) { calledQueries = append(calledQueries, sql) } rec3, _ := app.Dao().FindRecordById("users", "oap640cot4yru2s") diff --git a/pocketbase.go b/pocketbase.go index 409b8da9..90348a5b 100644 --- a/pocketbase.go +++ b/pocketbase.go @@ -148,12 +148,6 @@ func (pb *PocketBase) Start() error { // This method differs from pb.Start() by not registering the default // system commands! func (pb *PocketBase) Execute() error { - // Run the bootstrap process if the command exist and it is not - // the default cobra version command to prevent creating - // unnecessary the initialization system files. - // - // https://github.com/pocketbase/pocketbase/issues/404 - // https://github.com/pocketbase/pocketbase/discussions/1267 if !pb.skipBootstrap() { if err := pb.Bootstrap(); err != nil { return err @@ -218,11 +212,14 @@ func (pb *PocketBase) eagerParseFlags(config *Config) error { return pb.RootCmd.ParseFlags(os.Args[1:]) } -// eager check if the app should skip the bootstap process: +// skipBootstrap eagerly checks if the app should skip the bootstap process: // - already bootstrapped // - is unknown command // - is the default help command // - is the default version command +// +// https://github.com/pocketbase/pocketbase/issues/404 +// https://github.com/pocketbase/pocketbase/discussions/1267 func (pb *PocketBase) skipBootstrap() bool { flags := []string{ "-h", @@ -258,7 +255,7 @@ func (pb *PocketBase) skipBootstrap() bool { return false } -// tries to find the base executable directory and how it was run +// inspectRuntime tries to find the base executable directory and how it was run. func inspectRuntime() (baseDir string, withGoRun bool) { if strings.HasPrefix(os.Args[0], os.TempDir()) { // probably ran with go run