colin
/
pocketbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

speedup records cascade delete

This commit is contained in:
Gani Georgiev 2024-12-12 22:55:55 +02:00
parent efe4ef500b
commit 8e63e81561
5 changed files with 206 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -6,6 +6,8 @@
- Added `tests.NewTestAppWithConfig(config)` helper if you need more control over the test configurations like `IsDev`, the number of allowed connections, etc.
- Added `app.FindCachedCollectionReferences(collection, excludeIds)` to speedup records cascade delete almost twice for projects with many collections.
- ⚠️ Removed the "dry submit" when executing the collections Create API rule
(you can find more details why this change was introduced and how it could affect your app in https://github.com/pocketbase/pocketbase/discussions/6073).
For most users it should be non-breaking change, BUT if you have Create API rules that uses self-references or view counters you may have to adjust them manually.

33
core/app.go
View File

@ -372,13 +372,6 @@ type App interface {
// To manually reload the cache you can call [App.ReloadCachedCollections()]
FindCachedCollectionByNameOrId(nameOrId string) (*Collection, error)
// IsCollectionNameUnique checks that there is no existing collection
// with the provided name (case insensitive!).
//
// Note: case insensitive check because the name is used also as
// table name for the records.
IsCollectionNameUnique(name string, excludeIds ...string) bool
// FindCollectionReferences returns information for all relation
// fields referencing the provided collection.
//
@ -387,6 +380,32 @@ type App interface {
// as the excludeIds argument.
FindCollectionReferences(collection *Collection, excludeIds ...string) (map[*Collection][]Field, error)
// FindCachedCollectionReferences is similar to [App.FindCollectionReferences]
// but retrieves the Collection from the app cache instead of making a db call.
//
// NB! This method is suitable for read-only Collection operations.
//
// If you plan making changes to the returned Collection model,
// use [App.FindCollectionReferences] instead.
//
// Caveats:
//
// - The returned Collection should be used only for read-only operations.
// Avoid directly modifying the returned cached Collection as it will affect
// the global cached value even if you don't persist the changes in the database!
// - If you are updating a Collection in a transaction and then call this method before commit,
// it'll return the cached Collection state and not the one from the uncommitted transaction.
// - The cache is automatically updated on collections db change (create/update/delete).
// To manually reload the cache you can call [App.ReloadCachedCollections()].
FindCachedCollectionReferences(collection *Collection, excludeIds ...string) (map[*Collection][]Field, error)
// IsCollectionNameUnique checks that there is no existing collection
// with the provided name (case insensitive!).
//
// Note: case insensitive check because the name is used also as
// table name for the records.
IsCollectionNameUnique(name string, excludeIds ...string) bool
// TruncateCollection deletes all records associated with the provided collection.
//
// The truncate operation is executed in a single transaction,

93
core/collection_query.go
View File

@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"github.com/pocketbase/dbx"
@ -94,7 +95,7 @@ func (app *BaseApp) FindCollectionByNameOrId(nameOrId string) (*Collection, erro
// - If you are updating a Collection in a transaction and then call this method before commit,
// it'll return the cached Collection state and not the one from the uncommitted transaction.
// - The cache is automatically updated on collections db change (create/update/delete).
// To manually reload the cache you can call [App.ReloadCachedCollections()]
// To manually reload the cache you can call [App.ReloadCachedCollections()].
func (app *BaseApp) FindCachedCollectionByNameOrId(nameOrId string) (*Collection, error) {
collections, _ := app.Store().Get(StoreKeyCachedCollections).([]*Collection)
if collections == nil {
@ -111,30 +112,6 @@ func (app *BaseApp) FindCachedCollectionByNameOrId(nameOrId string) (*Collection
return nil, sql.ErrNoRows
}
// IsCollectionNameUnique checks that there is no existing collection
// with the provided name (case insensitive!).
//
// Note: case insensitive check because the name is used also as
// table name for the records.
func (app *BaseApp) IsCollectionNameUnique(name string, excludeIds ...string) bool {
if name == "" {
return false
}
query := app.CollectionQuery().
Select("count(*)").
AndWhere(dbx.NewExp("LOWER([[name]])={:name}", dbx.Params{"name": strings.ToLower(name)})).
Limit(1)
if uniqueExcludeIds := list.NonzeroUniques(excludeIds); len(uniqueExcludeIds) > 0 {
query.AndWhere(dbx.NotIn("id", list.ToInterfaceSlice(uniqueExcludeIds)...))
}
var exists bool
return query.Row(&exists) == nil && !exists
}
// FindCollectionReferences returns information for all relation fields
// referencing the provided collection.
//
@ -168,6 +145,72 @@ func (app *BaseApp) FindCollectionReferences(collection *Collection, excludeIds
return result, nil
}
// FindCachedCollectionReferences is similar to [App.FindCollectionReferences]
// but retrieves the Collection from the app cache instead of making a db call.
//
// NB! This method is suitable for read-only Collection operations.
//
// If you plan making changes to the returned Collection model,
// use [App.FindCollectionReferences] instead.
//
// Caveats:
//
// - The returned Collection should be used only for read-only operations.
// Avoid directly modifying the returned cached Collection as it will affect
// the global cached value even if you don't persist the changes in the database!
// - If you are updating a Collection in a transaction and then call this method before commit,
// it'll return the cached Collection state and not the one from the uncommitted transaction.
// - The cache is automatically updated on collections db change (create/update/delete).
// To manually reload the cache you can call [App.ReloadCachedCollections()].
func (app *BaseApp) FindCachedCollectionReferences(collection *Collection, excludeIds ...string) (map[*Collection][]Field, error) {
collections, _ := app.Store().Get(StoreKeyCachedCollections).([]*Collection)
if collections == nil {
// cache is not initialized yet (eg. run in a system migration)
return app.FindCollectionReferences(collection, excludeIds...)
}
result := map[*Collection][]Field{}
for _, c := range collections {
if slices.Contains(excludeIds, c.Id) {
continue
}
for _, rawField := range c.Fields {
f, ok := rawField.(*RelationField)
if ok && f.CollectionId == collection.Id {
result[c] = append(result[c], f)
}
}
}
return result, nil
}
// IsCollectionNameUnique checks that there is no existing collection
// with the provided name (case insensitive!).
//
// Note: case insensitive check because the name is used also as
// table name for the records.
func (app *BaseApp) IsCollectionNameUnique(name string, excludeIds ...string) bool {
if name == "" {
return false
}
query := app.CollectionQuery().
Select("count(*)").
AndWhere(dbx.NewExp("LOWER([[name]])={:name}", dbx.Params{"name": strings.ToLower(name)})).
Limit(1)
if uniqueExcludeIds := list.NonzeroUniques(excludeIds); len(uniqueExcludeIds) > 0 {
query.AndWhere(dbx.NotIn("id", list.ToInterfaceSlice(uniqueExcludeIds)...))
}
var exists bool
return query.Row(&exists) == nil && !exists
}
// TruncateCollection deletes all records associated with the provided collection.
//
// The truncate operation is executed in a single transaction,

137
core/collection_query_test.go
View File

@ -208,34 +208,6 @@ func TestFindCachedCollectionByNameOrId(t *testing.T) {
run(false)
}
func TestIsCollectionNameUnique(t *testing.T) {
t.Parallel()
app, _ := tests.NewTestApp()
defer app.Cleanup()
scenarios := []struct {
name string
excludeId string
expected bool
}{
{"", "", false},
{"demo1", "", false},
{"Demo1", "", false},
{"new", "", true},
{"demo1", "wsmn24bux7wo113", true},
}
for i, s := range scenarios {
t.Run(fmt.Sprintf("%d_%s", i, s.name), func(t *testing.T) {
result := app.IsCollectionNameUnique(s.name, s.excludeId)
if result != s.expected {
t.Errorf("Expected %v, got %v", s.expected, result)
}
})
}
}
func TestFindCollectionReferences(t *testing.T) {
t.Parallel()
@ -288,6 +260,115 @@ func TestFindCollectionReferences(t *testing.T) {
}
}
func TestFindCachedCollectionReferences(t *testing.T) {
t.Parallel()
app, _ := tests.NewTestApp()
defer app.Cleanup()
collection, err := app.FindCollectionByNameOrId("demo3")
if err != nil {
t.Fatal(err)
}
totalQueries := 0
app.DB().(*dbx.DB).QueryLogFunc = func(ctx context.Context, t time.Duration, sql string, rows *sql.Rows, err error) {
totalQueries++
}
run := func(withCache bool) {
var expectedTotalQueries int
if withCache {
err := app.ReloadCachedCollections()
if err != nil {
t.Fatal(err)
}
} else {
app.Store().Reset(nil)
expectedTotalQueries = 1
}
totalQueries = 0
result, err := app.FindCachedCollectionReferences(
collection,
collection.Id,
// test whether "nonempty" exclude ids condition will be skipped
"",
"",
)
if err != nil {
t.Fatal(err)
}
if len(result) != 1 {
t.Fatalf("Expected 1 collection, got %d: %v", len(result), result)
}
expectedFields := []string{
"rel_one_no_cascade",
"rel_one_no_cascade_required",
"rel_one_cascade",
"rel_one_unique",
"rel_many_no_cascade",
"rel_many_no_cascade_required",
"rel_many_cascade",
"rel_many_unique",
}
for col, fields := range result {
if col.Name != "demo4" {
t.Fatalf("Expected collection demo4, got %s", col.Name)
}
if len(fields) != len(expectedFields) {
t.Fatalf("Expected fields %v, got %v", expectedFields, fields)
}
for i, f := range fields {
if !slices.Contains(expectedFields, f.GetName()) {
t.Fatalf("[%d] Didn't expect field %v", i, f)
}
}
}
if totalQueries != expectedTotalQueries {
t.Fatalf("Expected %d totalQueries, got %d", expectedTotalQueries, totalQueries)
}
}
run(true)
run(false)
}
func TestIsCollectionNameUnique(t *testing.T) {
t.Parallel()
app, _ := tests.NewTestApp()
defer app.Cleanup()
scenarios := []struct {
name string
excludeId string
expected bool
}{
{"", "", false},
{"demo1", "", false},
{"Demo1", "", false},
{"new", "", true},
{"demo1", "wsmn24bux7wo113", true},
}
for i, s := range scenarios {
t.Run(fmt.Sprintf("%d_%s", i, s.name), func(t *testing.T) {
result := app.IsCollectionNameUnique(s.name, s.excludeId)
if result != s.expected {
t.Errorf("Expected %v, got %v", s.expected, result)
}
})
}
}
func TestFindCollectionTruncate(t *testing.T) {
t.Parallel()

2
core/record_model.go
View File

@ -1417,7 +1417,7 @@ func onRecordDeleteExecute(e *RecordEvent) error {
//
// note: the select is outside of the transaction to minimize
// SQLITE_BUSY errors when mixing read&write in a single transaction
refs, err := e.App.FindCollectionReferences(e.Record.Collection())
refs, err := e.App.FindCachedCollectionReferences(e.Record.Collection())
if err != nil {
return err
}