synced with master

CHANGELOG.md

@@ -85,6 +85,11 @@
 - (@todo docs) Added `record.ExpandedOne(rel)` and `record.ExpandedAll(rel)` helpers to retrieve casted single or multiple expand relations from the already loaded "expand" Record data.
 
 
+## v0.16.10
+
+- Added multiple valued fields (`relation`, `select`, `file`) normalizations to ensure that the zero-default value of a newly created multiple field is applied for already existing data ([#2930](https://github.com/pocketbase/pocketbase/issues/2930)).
+
+
 ## v0.16.9
 
 - Register the `eagerRequestDataCache` middleware only for the internal `api` group routes to avoid conflicts with custom route handlers ([#2914](https://github.com/pocketbase/pocketbase/issues/2914)).

daos/record_table_sync.go

@@ -174,9 +174,13 @@ func (dao *Dao) normalizeSingleVsMultipleFieldChanges(newCollection, oldCollecti
 
 	return dao.RunInTransaction(func(txDao *Dao) error {
 		for _, newField := range newCollection.Schema.Fields() {
-			oldField := oldCollection.Schema.GetFieldById(newField.Id)
+			// allow to continue even if there is no old field for the cases
-			if oldField == nil {
+			// when a new field is added and there are already inserted data
-				continue
+			var isOldMultiple bool
+			if oldField := oldCollection.Schema.GetFieldById(newField.Id); oldField != nil {
+				if opt, ok := oldField.Options.(schema.MultiValuer); ok {
+					isOldMultiple = opt.IsMultiple()
+				}
 			}
 
 			var isNewMultiple bool
@@ -184,11 +188,6 @@ func (dao *Dao) normalizeSingleVsMultipleFieldChanges(newCollection, oldCollecti
 				isNewMultiple = opt.IsMultiple()
 			}
 
-			var isOldMultiple bool
-			if opt, ok := oldField.Options.(schema.MultiValuer); ok {
-				isOldMultiple = opt.IsMultiple()
-			}
-
 			if isOldMultiple == isNewMultiple {
 				continue // no change
 			}

daos/record_table_sync_test.go

@@ -171,6 +171,18 @@ func TestSingleVsMultipleValuesNormalization(t *testing.T) {
 		opt := relManyField.Options.(*schema.RelationOptions)
 		opt.MaxSelect = types.Pointer(1)
 	}
+	{
+		// new multivaluer field to check whether the array normalization
+		// will be applied for already inserted data
+		collection.Schema.AddField(&schema.SchemaField{
+			Name: "new_multiple",
+			Type: schema.FieldTypeSelect,
+			Options: &schema.SelectOptions{
+				Values:    []string{"a", "b", "c"},
+				MaxSelect: 3,
+			},
+		})
+	}
 
 	if err := app.Dao().SaveCollection(collection); err != nil {
 		t.Fatal(err)
@@ -183,6 +195,7 @@ func TestSingleVsMultipleValuesNormalization(t *testing.T) {
 		FileMany    string `db:"file_many"`
 		RelOne      string `db:"rel_one"`
 		RelMany     string `db:"rel_many"`
+		NewMultiple string `db:"new_multiple"`
 	}
 
 	scenarios := []struct {
@@ -198,6 +211,7 @@ func TestSingleVsMultipleValuesNormalization(t *testing.T) {
 				FileMany:    ``,
 				RelOne:      `[]`,
 				RelMany:     ``,
+				NewMultiple: `[]`,
 			},
 		},
 		{
@@ -209,6 +223,7 @@ func TestSingleVsMultipleValuesNormalization(t *testing.T) {
 				FileMany:    ``,
 				RelOne:      `["84nmscqy84lsi1t"]`,
 				RelMany:     `oap640cot4yru2s`,
+				NewMultiple: `[]`,
 			},
 		},
 		{
@@ -220,6 +235,7 @@ func TestSingleVsMultipleValuesNormalization(t *testing.T) {
 				FileMany:    `test_tC1Yc87DfC.txt`,
 				RelOne:      `[]`,
 				RelMany:     `oap640cot4yru2s`,
+				NewMultiple: `[]`,
 			},
 		},
 	}
@@ -234,6 +250,7 @@ func TestSingleVsMultipleValuesNormalization(t *testing.T) {
 			"file_many",
 			"rel_one",
 			"rel_many",
+			"new_multiple",
 		).From(collection.Name).Where(dbx.HashExp{"id": s.recordId}).One(result)
 		if err != nil {
 			t.Errorf("[%s] Failed to load record: %v", s.recordId, err)

migrations/1679943780_normalize_single_multiple_values.go

@@ -12,6 +12,13 @@ import (
 // Normalizes old single and multiple values of MultiValuer fields (file, select, relation).
 func init() {
 	AppMigrations.Register(func(db dbx.Builder) error {
+		return normalizeMultivaluerFields(db)
+	}, func(db dbx.Builder) error {
+		return nil
+	})
+}
+
+func normalizeMultivaluerFields(db dbx.Builder) error {
 	dao := daos.New(db)
 
 	collections := []*models.Collection{}
@@ -98,7 +105,4 @@ func init() {
 	}
 
 	return nil
-	}, func(db dbx.Builder) error {
-		return nil
-	})
 }

migrations/1689579878_renormalize_single_multiple_values.go

@@ -0,0 +1,15 @@
+package migrations
+
+import (
+	"github.com/pocketbase/dbx"
+)
+
+// Renormalizes old single and multiple values of MultiValuer fields (file, select, relation)
+// (see https://github.com/pocketbase/pocketbase/issues/2930).
+func init() {
+	AppMigrations.Register(func(db dbx.Builder) error {
+		return normalizeMultivaluerFields(db)
+	}, func(db dbx.Builder) error {
+		return nil
+	})
+}

resolvers/record_field_resolve_runner.go

@@ -580,8 +580,8 @@ func (r *runner) processActiveProps() (*search.ResolverResult, error) {
 func jsonArrayLength(tableColumnPair string) string {
 	return fmt.Sprintf(
 		// note: the case is used to normalize value access for single and multiple relations.
-		`json_array_length(CASE WHEN json_valid([[%s]]) THEN [[%s]] ELSE json_array([[%s]]) END)`,
+		`json_array_length(CASE WHEN json_valid([[%s]]) THEN [[%s]] ELSE (CASE WHEN [[%s]] = '' OR [[%s]] IS NULL THEN json_array() ELSE json_array([[%s]]) END) END)`,
-		tableColumnPair, tableColumnPair, tableColumnPair,
+		tableColumnPair, tableColumnPair, tableColumnPair, tableColumnPair, tableColumnPair,
 	)
 }
 

resolvers/record_field_resolver_test.go

@@ -294,14 +294,14 @@ func TestRecordFieldResolverUpdateQuery(t *testing.T) {
 				"self_rel_one.rel_many_cascade.files:length != 7 &&" +
 				"self_rel_one.rel_many_cascade.files:length ?!= 8",
 			false,
-			"SELECT DISTINCT `demo4`.* FROM `demo4` LEFT JOIN `demo4` `__data_demo4` ON [[__data_demo4.id]]={:TEST} LEFT JOIN `demo3` `__data_demo3` ON [[__data_demo3.id]] IN ({:TEST}, {:TEST}) LEFT JOIN `demo4` `demo4_self_rel_one` ON [[demo4_self_rel_one.id]] = [[demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[demo4_self_rel_one.rel_many_cascade]]) THEN [[demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[demo4_self_rel_one.rel_many_cascade]]) END) `demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `demo4_self_rel_one_rel_many_cascade` ON [[demo4_self_rel_one_rel_many_cascade.id]] = [[demo4_self_rel_one_rel_many_cascade_je.value]] WHERE (json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE json_array([[__data_demo4.self_rel_many]]) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE json_array([[__data_demo4.self_rel_many]]) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE json_array([[__data_demo3.files]]) END) < {:TEST} AND ((json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE json_array([[__data_demo3.files]]) END) < {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__data_mm_demo3.files]]) THEN [[__data_mm_demo3.files]] ELSE json_array([[__data_mm_demo3.files]]) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo3` `__data_mm_demo3` ON `__data_mm_demo3`.`id` IN ({:TEST}, {:TEST}) WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] < {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END) = {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END) = {:TEST} AND ((json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END) != {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) THEN [[__mm_demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo4` `__mm_demo4_self_rel_one` ON [[__mm_demo4_self_rel_one.id]] = [[__mm_demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo4_self_rel_one.rel_many_cascade]]) THEN [[__mm_demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[__mm_demo4_self_rel_one.rel_many_cascade]]) END) `__mm_demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `__mm_demo4_self_rel_one_rel_many_cascade` ON [[__mm_demo4_self_rel_one_rel_many_cascade.id]] = [[__mm_demo4_self_rel_one_rel_many_cascade_je.value]] WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] != {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END) != {:TEST})",
+			"SELECT DISTINCT `demo4`.* FROM `demo4` LEFT JOIN `demo4` `__data_demo4` ON [[__data_demo4.id]]={:TEST} LEFT JOIN `demo3` `__data_demo3` ON [[__data_demo3.id]] IN ({:TEST}, {:TEST}) LEFT JOIN `demo4` `demo4_self_rel_one` ON [[demo4_self_rel_one.id]] = [[demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[demo4_self_rel_one.rel_many_cascade]]) THEN [[demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[demo4_self_rel_one.rel_many_cascade]]) END) `demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `demo4_self_rel_one_rel_many_cascade` ON [[demo4_self_rel_one_rel_many_cascade.id]] = [[demo4_self_rel_one_rel_many_cascade_je.value]] WHERE (json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE (CASE WHEN [[__data_demo4.self_rel_many]] = '' OR [[__data_demo4.self_rel_many]] IS NULL THEN json_array() ELSE json_array([[__data_demo4.self_rel_many]]) END) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo4.self_rel_many]]) THEN [[__data_demo4.self_rel_many]] ELSE (CASE WHEN [[__data_demo4.self_rel_many]] = '' OR [[__data_demo4.self_rel_many]] IS NULL THEN json_array() ELSE json_array([[__data_demo4.self_rel_many]]) END) END) > {:TEST} AND json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE (CASE WHEN [[__data_demo3.files]] = '' OR [[__data_demo3.files]] IS NULL THEN json_array() ELSE json_array([[__data_demo3.files]]) END) END) < {:TEST} AND ((json_array_length(CASE WHEN json_valid([[__data_demo3.files]]) THEN [[__data_demo3.files]] ELSE (CASE WHEN [[__data_demo3.files]] = '' OR [[__data_demo3.files]] IS NULL THEN json_array() ELSE json_array([[__data_demo3.files]]) END) END) < {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__data_mm_demo3.files]]) THEN [[__data_mm_demo3.files]] ELSE (CASE WHEN [[__data_mm_demo3.files]] = '' OR [[__data_mm_demo3.files]] IS NULL THEN json_array() ELSE json_array([[__data_mm_demo3.files]]) END) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo3` `__data_mm_demo3` ON `__data_mm_demo3`.`id` IN ({:TEST}, {:TEST}) WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] < {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE (CASE WHEN [[demo4_self_rel_one.self_rel_many]] = '' OR [[demo4_self_rel_one.self_rel_many]] IS NULL THEN json_array() ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END) END) = {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one.self_rel_many]]) THEN [[demo4_self_rel_one.self_rel_many]] ELSE (CASE WHEN [[demo4_self_rel_one.self_rel_many]] = '' OR [[demo4_self_rel_one.self_rel_many]] IS NULL THEN json_array() ELSE json_array([[demo4_self_rel_one.self_rel_many]]) END) END) = {:TEST} AND ((json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE (CASE WHEN [[demo4_self_rel_one_rel_many_cascade.files]] = '' OR [[demo4_self_rel_one_rel_many_cascade.files]] IS NULL THEN json_array() ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END) END) != {:TEST}) AND (NOT EXISTS (SELECT 1 FROM (SELECT json_array_length(CASE WHEN json_valid([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) THEN [[__mm_demo4_self_rel_one_rel_many_cascade.files]] ELSE (CASE WHEN [[__mm_demo4_self_rel_one_rel_many_cascade.files]] = '' OR [[__mm_demo4_self_rel_one_rel_many_cascade.files]] IS NULL THEN json_array() ELSE json_array([[__mm_demo4_self_rel_one_rel_many_cascade.files]]) END) END) as [[multiMatchValue]] FROM `demo4` `__mm_demo4` LEFT JOIN `demo4` `__mm_demo4_self_rel_one` ON [[__mm_demo4_self_rel_one.id]] = [[__mm_demo4.self_rel_one]] LEFT JOIN json_each(CASE WHEN json_valid([[__mm_demo4_self_rel_one.rel_many_cascade]]) THEN [[__mm_demo4_self_rel_one.rel_many_cascade]] ELSE json_array([[__mm_demo4_self_rel_one.rel_many_cascade]]) END) `__mm_demo4_self_rel_one_rel_many_cascade_je` LEFT JOIN `demo3` `__mm_demo4_self_rel_one_rel_many_cascade` ON [[__mm_demo4_self_rel_one_rel_many_cascade.id]] = [[__mm_demo4_self_rel_one_rel_many_cascade_je.value]] WHERE `__mm_demo4`.`id` = `demo4`.`id`) {{__smTEST}} WHERE ((NOT ([[__smTEST.multiMatchValue]] != {:TEST})) OR ([[__smTEST.multiMatchValue]] IS NULL))))) AND json_array_length(CASE WHEN json_valid([[demo4_self_rel_one_rel_many_cascade.files]]) THEN [[demo4_self_rel_one_rel_many_cascade.files]] ELSE (CASE WHEN [[demo4_self_rel_one_rel_many_cascade.files]] = '' OR [[demo4_self_rel_one_rel_many_cascade.files]] IS NULL THEN json_array() ELSE json_array([[demo4_self_rel_one_rel_many_cascade.files]]) END) END) != {:TEST})",
 		},
 		{
 			"json_extract and json_array_length COALESCE equal normalizations",
 			"demo4",
 			"json_object.a.b = '' && self_rel_many:length != 2 && json_object.a.b > 3 && self_rel_many:length <= 4",
 			false,
-			"SELECT `demo4`.* FROM `demo4` WHERE ((JSON_EXTRACT([[demo4.json_object]], '$.a.b') = '' OR JSON_EXTRACT([[demo4.json_object]], '$.a.b') IS NULL) AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE json_array([[demo4.self_rel_many]]) END) != {:TEST} AND JSON_EXTRACT([[demo4.json_object]], '$.a.b') > {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE json_array([[demo4.self_rel_many]]) END) <= {:TEST})",
+			"SELECT `demo4`.* FROM `demo4` WHERE ((JSON_EXTRACT([[demo4.json_object]], '$.a.b') = '' OR JSON_EXTRACT([[demo4.json_object]], '$.a.b') IS NULL) AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE (CASE WHEN [[demo4.self_rel_many]] = '' OR [[demo4.self_rel_many]] IS NULL THEN json_array() ELSE json_array([[demo4.self_rel_many]]) END) END) != {:TEST} AND JSON_EXTRACT([[demo4.json_object]], '$.a.b') > {:TEST} AND json_array_length(CASE WHEN json_valid([[demo4.self_rel_many]]) THEN [[demo4.self_rel_many]] ELSE (CASE WHEN [[demo4.self_rel_many]] = '' OR [[demo4.self_rel_many]] IS NULL THEN json_array() ELSE json_array([[demo4.self_rel_many]]) END) END) <= {:TEST})",
 		},
 	}