updated collection indexes on system fields validator and normalized v0.23 old collections migration

CHANGELOG.md

@@ -1,3 +1,11 @@
+## v0.23.0-rc14 (WIP)
+
+> [!CAUTION]
+> **This is a prerelease intended for test and experimental purposes only!**
+
+- Allow changing collate, sort and partial constraints for indexes on system fields.
+
+
 ## v0.23.0-rc13
 
 > [!CAUTION]

core/collection_import.go

@@ -86,6 +86,12 @@ func (app *BaseApp) ImportCollections(toImport []map[string]any, deleteMissing b
 					continue
 				}
 				if imported.Fields.GetById(f.GetId()) == nil {
+					// replace with the existing id to prevent accidental column deletion
+					// since otherwise the imported field will be treated as a new one
+					found := imported.Fields.GetByName(f.GetName())
+					if found != nil && found.Type() == f.Type() {
+						found.SetId(f.GetId())
+					}
 					imported.Fields.Add(f)
 				}
 			}

core/collection_validate.go

@@ -608,6 +608,12 @@ func (cv *collectionValidator) checkIndexes(value any) error {
 				continue
 			}
 
+			// reset collate and sort since they are not important for the unique constraint
+			for i := range oldParsed.Columns {
+				oldParsed.Columns[i].Collate = ""
+				oldParsed.Columns[i].Sort = ""
+			}
+
 			oldParsedStr := oldParsed.Build()
 
 			for _, column := range oldParsed.Columns {
@@ -621,19 +627,29 @@ func (cv *collectionValidator) checkIndexes(value any) error {
 						newParsed := dbutils.ParseIndex(newIndex)
 
 						// exclude the non-important identifiers from the check
+						newParsed.SchemaName = oldParsed.SchemaName
 						newParsed.IndexName = oldParsed.IndexName
 						newParsed.TableName = oldParsed.TableName
 
+						// exclude partial constraints
+						newParsed.Where = oldParsed.Where
+
+						// reset collate and sort
+						for i := range newParsed.Columns {
+							newParsed.Columns[i].Collate = ""
+							newParsed.Columns[i].Sort = ""
+						}
+
 						if oldParsedStr == newParsed.Build() {
 							hasMatch = true
-							continue
+							break
 						}
 					}
 
 					if !hasMatch {
 						return validation.NewError(
-							"validation_unique_system_field_index_change",
+							"validation_invalid_unique_system_field_index",
-							fmt.Sprintf("Unique index definition on system fields (%q) cannot be changed.", f.GetName()),
+							fmt.Sprintf("Unique index definition on system fields (%q) is invalid or missing.", f.GetName()),
 						).SetParams(map[string]any{"fieldName": f.GetName()})
 					}
 

core/collection_validate_test.go

@@ -550,7 +550,7 @@ func TestCollectionValidate(t *testing.T) {
 			expectedErrors: []string{"indexes"},
 		},
 		{
-			name: "changing index on system field",
+			name: "changing partial constraint of existing index on system field",
 			collection: func(app core.App) (*core.Collection, error) {
 				demo2, err := app.FindCollectionByNameOrId("demo2")
 				if err != nil {
@@ -571,7 +571,91 @@ func TestCollectionValidate(t *testing.T) {
 
 				// replace the index with a partial one
 				demo2.RemoveIndex("idx_unique_demo2_title")
-				demo2.AddIndex("idx_unique_demo2_title", true, "title", "1 = 1")
+				demo2.AddIndex("idx_new_demo2_title", true, "title", "1 = 1")
+
+				return demo2, nil
+			},
+			expectedErrors: []string{},
+		},
+		{
+			name: "changing column sort and collate of existing index on system field",
+			collection: func(app core.App) (*core.Collection, error) {
+				demo2, err := app.FindCollectionByNameOrId("demo2")
+				if err != nil {
+					return nil, err
+				}
+
+				// mark the title field as system
+				demo2.Fields.GetByName("title").SetSystem(true)
+				if err = app.Save(demo2); err != nil {
+					return nil, err
+				}
+
+				// refresh
+				demo2, err = app.FindCollectionByNameOrId("demo2")
+				if err != nil {
+					return nil, err
+				}
+
+				// replace the index with a new one for the same column but with collate and sort
+				demo2.RemoveIndex("idx_unique_demo2_title")
+				demo2.AddIndex("idx_new_demo2_title", true, "title COLLATE test ASC", "")
+
+				return demo2, nil
+			},
+			expectedErrors: []string{},
+		},
+		{
+			name: "adding new column to index on system field",
+			collection: func(app core.App) (*core.Collection, error) {
+				demo2, err := app.FindCollectionByNameOrId("demo2")
+				if err != nil {
+					return nil, err
+				}
+
+				// mark the title field as system
+				demo2.Fields.GetByName("title").SetSystem(true)
+				if err = app.Save(demo2); err != nil {
+					return nil, err
+				}
+
+				// refresh
+				demo2, err = app.FindCollectionByNameOrId("demo2")
+				if err != nil {
+					return nil, err
+				}
+
+				// replace the index with a non-unique one
+				demo2.RemoveIndex("idx_unique_demo2_title")
+				demo2.AddIndex("idx_new_title", false, "title, id", "")
+
+				return demo2, nil
+			},
+			expectedErrors: []string{"indexes"},
+		},
+		{
+			name: "changing index type on system field",
+			collection: func(app core.App) (*core.Collection, error) {
+				demo2, err := app.FindCollectionByNameOrId("demo2")
+				if err != nil {
+					return nil, err
+				}
+
+				// mark the title field as system
+				demo2.Fields.GetByName("title").SetSystem(true)
+				if err = app.Save(demo2); err != nil {
+					return nil, err
+				}
+
+				// refresh
+				demo2, err = app.FindCollectionByNameOrId("demo2")
+				if err != nil {
+					return nil, err
+				}
+
+				// replace the index with a non-unique one (partial constraints are ignored)
+				demo2.RemoveIndex("idx_unique_demo2_title")
+				demo2.AddIndex("idx_new_title", false, "title", "1=1")
 
 				return demo2, nil
 			},

migrations/1717233556_v0.23_migrate.go

@@ -365,8 +365,10 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 
 			c.Options["manageRule"] = nil
 			if options["manageRule"] != nil {
-				manageRule := cast.ToString(options["manageRule"])
+				manageRule, err := cast.ToStringE(options["manageRule"])
-				c.Options["manageRule"] = &manageRule
+				if err == nil && manageRule != "" {
+					c.Options["manageRule"] = migrateRule(&manageRule)
+				}
 			}
 
 			// passwordAuth
@@ -494,15 +496,15 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 			// ---
 			c.Indexes = append(types.JSONArray[string]{
 				fmt.Sprintf("CREATE UNIQUE INDEX `_%s_username_idx` ON `%s` (username COLLATE NOCASE)", c.Id, c.Name),
-				fmt.Sprintf("CREATE UNIQUE INDEX `_%s_email_idx` ON `%s` (email) WHERE email != ''", c.Id, c.Name),
+				fmt.Sprintf("CREATE UNIQUE INDEX `_%s_email_idx` ON `%s` (`email`) WHERE `email` != ''", c.Id, c.Name),
-				fmt.Sprintf("CREATE UNIQUE INDEX `_%s_tokenKey_idx` ON `%s` (tokenKey)", c.Id, c.Name),
+				fmt.Sprintf("CREATE UNIQUE INDEX `_%s_tokenKey_idx` ON `%s` (`tokenKey`)", c.Id, c.Name),
 			}, c.Indexes...)
 
 			// prepend the auth system fields
 			// ---
 			tokenKeyField := map[string]any{
+				"id":                  fieldIdChecksum("text", "tokenKey"),
 				"type":                "text",
-				"id":                  "_pbf_auth_tokenKey_",
 				"name":                "tokenKey",
 				"system":              true,
 				"hidden":              true,
@@ -515,8 +517,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 				"autogeneratePattern": "[a-zA-Z0-9_]{50}",
 			}
 			passwordField := map[string]any{
+				"id":          fieldIdChecksum("password", "password"),
 				"type":        "password",
-				"id":          "_pbf_auth_password_",
 				"name":        "password",
 				"presentable": false,
 				"system":      true,
@@ -527,8 +529,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 				"cost":        bcrypt.DefaultCost, // new default
 			}
 			emailField := map[string]any{
+				"id":            fieldIdChecksum("email", "email"),
 				"type":          "email",
-				"id":            "_pbf_auth_email_",
 				"name":          "email",
 				"system":        true,
 				"hidden":        false,
@@ -538,8 +540,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 				"onlyDomains":   cast.ToStringSlice(options["onlyEmailDomains"]),
 			}
 			emailVisibilityField := map[string]any{
+				"id":          fieldIdChecksum("bool", "emailVisibility"),
 				"type":        "bool",
-				"id":          "_pbf_auth_emailVisibility_",
 				"name":        "emailVisibility",
 				"system":      true,
 				"hidden":      false,
@@ -547,8 +549,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 				"required":    false,
 			}
 			verifiedField := map[string]any{
+				"id":          fieldIdChecksum("bool", "verified"),
 				"type":        "bool",
-				"id":          "_pbf_auth_verified_",
 				"name":        "verified",
 				"system":      true,
 				"hidden":      false,
@@ -556,8 +558,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 				"required":    false,
 			}
 			usernameField := map[string]any{
+				"id":                  fieldIdChecksum("text", "username"),
 				"type":                "text",
-				"id":                  "_pbf_auth_username_",
 				"name":                "username",
 				"system":              false,
 				"hidden":              false,
@@ -597,8 +599,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 
 		// prepend the id field
 		idField := map[string]any{
+			"id":                  fieldIdChecksum("text", "id"),
 			"type":                "text",
-			"id":                  "_pbf_text_id_",
 			"name":                "id",
 			"system":              true,
 			"required":            true,
@@ -631,8 +633,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 
 		if addCreated {
 			createdField := map[string]any{
+				"id":          fieldIdChecksum("autodate", "created"),
 				"type":        "autodate",
-				"id":          "_pbf_autodate_created_",
 				"name":        "created",
 				"system":      false,
 				"presentable": false,
@@ -645,8 +647,8 @@ func migrateOldCollections(txApp core.App, oldSettings *oldSettingsModel) error
 
 		if addUpdated {
 			updatedField := map[string]any{
+				"id":          fieldIdChecksum("autodate", "updated"),
 				"type":        "autodate",
-				"id":          "_pbf_autodate_updated_",
 				"name":        "updated",
 				"system":      false,
 				"presentable": false,

migrations/1717233558_v0.23_migrate3.go

@@ -10,12 +10,12 @@ import (
 
 // note: this migration will be deleted in future version
 
-func collectionIdChecksum(c *core.Collection) string {
+func collectionIdChecksum(typ, name string) string {
-	return "pbc_" + strconv.Itoa(int(crc32.ChecksumIEEE([]byte(c.Type+c.Name))))
+	return "pbc_" + strconv.Itoa(int(crc32.ChecksumIEEE([]byte(typ+name))))
 }
 
-func fieldIdChecksum(f core.Field) string {
+func fieldIdChecksum(typ, name string) string {
-	return f.Type() + strconv.Itoa(int(crc32.ChecksumIEEE([]byte(f.GetName()))))
+	return typ + strconv.Itoa(int(crc32.ChecksumIEEE([]byte(name))))
 }
 
 // normalize system collection and field ids
@@ -62,7 +62,7 @@ func init() {
 			originalId := c.Id
 
 			// normalize collection id
-			if checksum := collectionIdChecksum(c); c.Id != checksum {
+			if checksum := collectionIdChecksum(c.Type, c.Name); c.Id != checksum {
 				c.Id = checksum
 				needUpdate = true
 			}
@@ -73,7 +73,7 @@ func init() {
 					continue
 				}
 
-				if checksum := fieldIdChecksum(f); f.GetId() != checksum {
+				if checksum := fieldIdChecksum(f.Type(), f.GetName()); f.GetId() != checksum {
 					f.SetId(checksum)
 					needUpdate = true
 				}