updated dao fail/retry handling

CHANGELOG.md

@@ -2,9 +2,11 @@
 
 - Added new "View" collection type (@todo document)
 
-- Added auto fail/retry for the `SELECT` queries to gracefully handle the `database is locked` errors ([#1795](https://github.com/pocketbase/pocketbase/discussions/1795#discussioncomment-4882169)).
+- Added auto fail/retry (default to 8 attempts) for the `SELECT` queries to gracefully handle the `database is locked` errors ([#1795](https://github.com/pocketbase/pocketbase/discussions/1795#discussioncomment-4882169)).
+  _The default max attempts can be accessed or changed via `Dao.MaxLockRetries`._
 
-- Added default max query executation timeout (120s).
+- Added default max query executation timeout (90s).
+  _The default timeout can be access or changed via `Dao.ModelQueryTimeout`._
 
 - Added support for `dao.RecordQuery(collection)` to scan directly the `One()` and `All()` results in `*models.Record` or `[]*models.Record` without the need of explicit `NullStringMap`.
 
@@ -14,6 +16,10 @@
 
 - Enabled `process.env` in JS migrations to allow accessing `os.Environ()`.
 
+- Added `UploadedFiles` field to the `RecordCreateEvent` and `RecordUpdateEvent` event structs.
+
+- **!** Moved file upload after the record persistent to allow custom changing the record id safely from the `OnModelBeforeCreate` hook.
+
 - **!** Changed `System.GetFile()` to return directly `*blob.Reader` instead of the `io.ReadCloser` interface.
 
 - **!** Changed `To`, `Cc` and `Bcc` of `mailer.Message` to `[]mail.Address` for consistency and to allow multiple recipients and optional name.
@@ -38,6 +44,7 @@
 
 - **!** Removed the previously deprecated `Dao.Block()` and `Dao.Continue()` helpers in favor of `Dao.NonconcurrentDB()`.
 
+- Other minor Admin UI improvements.
 
 ## v0.12.3
 

daos/base.go

@@ -5,6 +5,7 @@ package daos
 
 import (
 	"errors"
+	"time"
 
 	"github.com/pocketbase/dbx"
 	"github.com/pocketbase/pocketbase/models"
@@ -20,8 +21,10 @@ func New(db dbx.Builder) *Dao {
 // async and sync db builders.
 func NewMultiDB(concurrentDB, nonconcurrentDB dbx.Builder) *Dao {
 	return &Dao{
-		concurrentDB:    concurrentDB,
+		concurrentDB:      concurrentDB,
-		nonconcurrentDB: nonconcurrentDB,
+		nonconcurrentDB:   nonconcurrentDB,
+		MaxLockRetries:    8,
+		ModelQueryTimeout: 90 * time.Second,
 	}
 }
 
@@ -32,6 +35,14 @@ type Dao struct {
 	concurrentDB    dbx.Builder
 	nonconcurrentDB dbx.Builder
 
+	// MaxLockRetries specifies the default max "database is locked" auto retry attempts.
+	MaxLockRetries int
+
+	// ModelQueryTimeout is the default max duration of a running ModelQuery().
+	//
+	// This field has no effect if an explicit query context is already specified.
+	ModelQueryTimeout time.Duration
+
 	BeforeCreateFunc func(eventDao *Dao, m models.Model) error
 	AfterCreateFunc  func(eventDao *Dao, m models.Model)
 	BeforeUpdateFunc func(eventDao *Dao, m models.Model) error
@@ -63,15 +74,17 @@ func (dao *Dao) NonconcurrentDB() dbx.Builder {
 	return dao.nonconcurrentDB
 }
 
-// ModelQuery creates a new query with preset Select and From fields
+// ModelQuery creates a new preconfigured select query with preset
-// based on the provided model argument.
+// SELECT, FROM and other common fields based on the provided model.
 func (dao *Dao) ModelQuery(m models.Model) *dbx.SelectQuery {
 	tableName := m.TableName()
 
 	return dao.DB().
 		Select("{{" + tableName + "}}.*").
 		From(tableName).
-		WithExecHook(onLockErrorRetry)
+		WithBuildHook(func(query *dbx.Query) {
+			query.WithExecHook(execLockRetry(dao.ModelQueryTimeout, dao.MaxLockRetries))
+		})
 }
 
 // FindById finds a single db record with the specified id and
@@ -189,7 +202,7 @@ func (dao *Dao) Delete(m models.Model) error {
 		}
 
 		return nil
-	}, defaultMaxRetries)
+	})
 }
 
 // Save upserts (update or create if primary key is not set) the provided model.
@@ -197,12 +210,12 @@ func (dao *Dao) Save(m models.Model) error {
 	if m.IsNew() {
 		return dao.lockRetry(func(retryDao *Dao) error {
 			return retryDao.create(m)
-		}, defaultMaxRetries)
+		})
 	}
 
 	return dao.lockRetry(func(retryDao *Dao) error {
 		return retryDao.update(m)
-	}, defaultMaxRetries)
+	})
 }
 
 func (dao *Dao) update(m models.Model) error {
@@ -296,7 +309,7 @@ func (dao *Dao) create(m models.Model) error {
 	return nil
 }
 
-func (dao *Dao) lockRetry(op func(retryDao *Dao) error, maxRetries int) error {
+func (dao *Dao) lockRetry(op func(retryDao *Dao) error) error {
 	retryDao := dao
 
 	return baseLockRetry(func(attempt int) error {
@@ -310,5 +323,5 @@ func (dao *Dao) lockRetry(op func(retryDao *Dao) error, maxRetries int) error {
 		}
 
 		return op(retryDao)
-	}, maxRetries)
+	}, dao.MaxLockRetries)
 }

daos/base_retry.go

@@ -8,26 +8,24 @@ import (
 	"github.com/pocketbase/dbx"
 )
 
-const defaultQueryTimeout time.Duration = 2 * time.Minute
+// default retries intervals (in ms)
+var defaultRetryIntervals = []int{100, 250, 350, 500, 700, 1000}
 
-const defaultMaxRetries int = 10
+func execLockRetry(timeout time.Duration, maxRetries int) dbx.ExecHookFunc {
-
+	return func(q *dbx.Query, op func() error) error {
-var defaultRetryIntervals = []int{100, 250, 350, 500, 700, 1000, 1200, 1500}
+		if q.Context() == nil {
-
+			cancelCtx, cancel := context.WithTimeout(context.Background(), timeout)
-func onLockErrorRetry(s *dbx.SelectQuery, op func() error) error {
-	return baseLockRetry(func(attempt int) error {
-		// load a default timeout context if not set explicitly
-		if s.Context() == nil {
-			ctx, cancel := context.WithTimeout(context.Background(), defaultQueryTimeout)
 			defer func() {
 				cancel()
-				s.WithContext(nil) // reset
+				q.WithContext(nil) // reset
 			}()
-			s.WithContext(ctx)
+			q.WithContext(cancelCtx)
 		}
 
-		return op()
+		return baseLockRetry(func(attempt int) error {
-	}, defaultMaxRetries)
+			return op()
+		}, maxRetries)
+	}
 }
 
 func baseLockRetry(op func(attempt int) error, maxRetries int) error {

daos/base_retry_test.go

@@ -6,12 +6,12 @@ import (
 )
 
 func TestGetDefaultRetryInterval(t *testing.T) {
-	if i := getDefaultRetryInterval(-1); i.Milliseconds() != 1500 {
+	if i := getDefaultRetryInterval(-1); i.Milliseconds() != 1000 {
-		t.Fatalf("Expected 1500ms, got %v", i)
+		t.Fatalf("Expected 1000ms, got %v", i)
 	}
 
-	if i := getDefaultRetryInterval(999); i.Milliseconds() != 1500 {
+	if i := getDefaultRetryInterval(999); i.Milliseconds() != 1000 {
-		t.Fatalf("Expected 1500ms, got %v", i)
+		t.Fatalf("Expected 1000ms, got %v", i)
 	}
 
 	if i := getDefaultRetryInterval(3); i.Milliseconds() != 500 {

daos/record.go

@@ -23,68 +23,70 @@ func (dao *Dao) RecordQuery(collection *models.Collection) *dbx.SelectQuery {
 	return dao.DB().
 		Select(selectCols).
 		From(tableName).
-		WithExecHook(onLockErrorRetry).
+		WithBuildHook(func(query *dbx.Query) {
-		WithOneHook(func(s *dbx.SelectQuery, a any, op func(b any) error) error {
+			query.WithExecHook(execLockRetry(dao.ModelQueryTimeout, dao.MaxLockRetries)).
-			switch v := a.(type) {
+				WithOneHook(func(q *dbx.Query, a any, op func(b any) error) error {
-			case *models.Record:
+					switch v := a.(type) {
-				if v == nil {
+					case *models.Record:
-					return op(a)
+						if v == nil {
-				}
+							return op(a)
+						}
 
-				row := dbx.NullStringMap{}
+						row := dbx.NullStringMap{}
-				if err := op(&row); err != nil {
+						if err := op(&row); err != nil {
-					return err
+							return err
-				}
+						}
 
-				record := models.NewRecordFromNullStringMap(collection, row)
+						record := models.NewRecordFromNullStringMap(collection, row)
 
-				*v = *record
+						*v = *record
 
-				return nil
+						return nil
-			default:
+					default:
-				return op(a)
+						return op(a)
-			}
+					}
-		}).
+				}).
-		WithAllHook(func(s *dbx.SelectQuery, sliceA any, op func(sliceB any) error) error {
+				WithAllHook(func(q *dbx.Query, sliceA any, op func(sliceB any) error) error {
-			switch v := sliceA.(type) {
+					switch v := sliceA.(type) {
-			case *[]*models.Record:
+					case *[]*models.Record:
-				if v == nil {
+						if v == nil {
-					return op(sliceA)
+							return op(sliceA)
-				}
+						}
 
-				rows := []dbx.NullStringMap{}
+						rows := []dbx.NullStringMap{}
-				if err := op(&rows); err != nil {
+						if err := op(&rows); err != nil {
-					return err
+							return err
-				}
+						}
 
-				records := models.NewRecordsFromNullStringMaps(collection, rows)
+						records := models.NewRecordsFromNullStringMaps(collection, rows)
 
-				*v = records
+						*v = records
 
-				return nil
+						return nil
-			case *[]models.Record:
+					case *[]models.Record:
-				if v == nil {
+						if v == nil {
-					return op(sliceA)
+							return op(sliceA)
-				}
+						}
 
-				rows := []dbx.NullStringMap{}
+						rows := []dbx.NullStringMap{}
-				if err := op(&rows); err != nil {
+						if err := op(&rows); err != nil {
-					return err
+							return err
-				}
+						}
 
-				records := models.NewRecordsFromNullStringMaps(collection, rows)
+						records := models.NewRecordsFromNullStringMaps(collection, rows)
 
-				nonPointers := make([]models.Record, len(records))
+						nonPointers := make([]models.Record, len(records))
-				for i, r := range records {
+						for i, r := range records {
-					nonPointers[i] = *r
+							nonPointers[i] = *r
-				}
+						}
 
-				*v = nonPointers
+						*v = nonPointers
 
-				return nil
+						return nil
-			default:
+					default:
-				return op(sliceA)
+						return op(sliceA)
-			}
+					}
+				})
 		})
 }
 

tools/search/provider.go

@@ -5,7 +5,6 @@ import (
 	"math"
 	"net/url"
 	"strconv"
-	"strings"
 
 	"github.com/pocketbase/dbx"
 )
@@ -198,11 +197,9 @@ func (s *Provider) Exec(items any) (*Result, error) {
 	if len(queryInfo.From) > 0 {
 		baseTable = queryInfo.From[0]
 	}
-	countQuery := modelsQuery
+	clone := modelsQuery
-	rawCountQuery := countQuery.Select(strings.Join([]string{baseTable, "id"}, ".")).OrderBy().Build().SQL()
+	countQuery := clone.Select("COUNT(DISTINCT {{" + baseTable + ".id}})").OrderBy()
-	wrappedCountQuery := queryInfo.Builder.NewQuery("SELECT COUNT(*) FROM (" + rawCountQuery + ")")
+	if err := countQuery.Row(&totalCount); err != nil {
-	wrappedCountQuery.Bind(countQuery.Build().Params())
-	if err := wrappedCountQuery.Row(&totalCount); err != nil {
 		return nil, err
 	}
 

tools/search/provider_test.go

@@ -228,7 +228,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":10,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(*) FROM (SELECT `test`.`id` FROM `test` WHERE NOT (`test1` IS NULL))",
+				"SELECT COUNT(DISTINCT {{test.id}}) FROM `test` WHERE NOT (`test1` IS NULL)",
 				"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 10",
 			},
 		},
@@ -241,7 +241,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":30,"totalItems":2,"totalPages":1,"items":[{"test1":1,"test2":"test2.1","test3":""},{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(*) FROM (SELECT `test`.`id` FROM `test` WHERE NOT (`test1` IS NULL))",
+				"SELECT COUNT(DISTINCT {{test.id}}) FROM `test` WHERE NOT (`test1` IS NULL)",
 				"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 30",
 			},
 		},
@@ -274,7 +274,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":` + fmt.Sprint(MaxPerPage) + `,"totalItems":1,"totalPages":1,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(*) FROM (SELECT `test`.`id` FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (COALESCE(test2, '') != COALESCE(null, ''))) AND (test1 >= 2))",
+				"SELECT COUNT(DISTINCT {{test.id}}) FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (COALESCE(test2, '') != COALESCE(null, ''))) AND (test1 >= 2)",
 				"SELECT * FROM `test` WHERE ((NOT (`test1` IS NULL)) AND (COALESCE(test2, '') != COALESCE(null, ''))) AND (test1 >= 2) ORDER BY `test1` ASC, `test2` DESC LIMIT 500",
 			},
 		},
@@ -287,7 +287,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":1,"perPage":10,"totalItems":0,"totalPages":0,"items":[]}`,
 			[]string{
-				"SELECT COUNT(*) FROM (SELECT `test`.`id` FROM `test` WHERE (NOT (`test1` IS NULL)) AND (COALESCE(test3, '') != COALESCE('', '')))",
+				"SELECT COUNT(DISTINCT {{test.id}}) FROM `test` WHERE (NOT (`test1` IS NULL)) AND (COALESCE(test3, '') != COALESCE('', ''))",
 				"SELECT * FROM `test` WHERE (NOT (`test1` IS NULL)) AND (COALESCE(test3, '') != COALESCE('', '')) ORDER BY `test1` ASC, `test3` ASC LIMIT 10",
 			},
 		},
@@ -300,7 +300,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 			false,
 			`{"page":2,"perPage":1,"totalItems":2,"totalPages":2,"items":[{"test1":2,"test2":"test2.2","test3":""}]}`,
 			[]string{
-				"SELECT COUNT(*) FROM (SELECT `test`.`id` FROM `test` WHERE NOT (`test1` IS NULL))",
+				"SELECT COUNT(DISTINCT {{test.id}}) FROM `test` WHERE NOT (`test1` IS NULL)",
 				"SELECT * FROM `test` WHERE NOT (`test1` IS NULL) ORDER BY `test1` ASC LIMIT 1 OFFSET 1",
 			},
 		},
@@ -345,7 +345,7 @@ func TestProviderExecNonEmptyQuery(t *testing.T) {
 
 		for _, q := range testDB.CalledQueries {
 			if !list.ExistInSliceWithRegex(q, s.expectQueries) {
-				t.Errorf("[%s] Didn't expect query \n%v in \n%v", s.name, q, testDB.CalledQueries)
+				t.Fatalf("[%s] Didn't expect query \n%v \nin \n%v", s.name, q, s.expectQueries)
 			}
 		}
 	}