From c0a7d0f6c0fed9c73bf9e38030569487f339f745 Mon Sep 17 00:00:00 2001 From: Gani Georgiev Date: Tue, 25 Apr 2023 17:58:51 +0300 Subject: [PATCH] added ?fields query parameter support to limit the returned api fields --- CHANGELOG.md | 11 ++ apis/base.go | 4 + apis/record_helpers.go | 4 +- tools/rest/json_serializer.go | 121 ++++++++++++++++++ tools/rest/json_serializer_test.go | 195 +++++++++++++++++++++++++++++ 5 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 tools/rest/json_serializer.go create mode 100644 tools/rest/json_serializer_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ce1e26b..03532232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ ## (WIP) +- Added option to limit the returned API fields using the `?fields` query parameter. + The "fields picker" is applied for `SearchResult.Items` and any other JSON response. For example: + ```js + // original: {"id": "RECORD_ID", "name": "abc", "description": "...something very big...", "items": ["id1", "id2"], "expand": {"items": [{"id": "id1", "name": "test1"}, {"id": "id2", "name": "test2"}]}} + // output: {"name": "abc", "expand": {"items": [{"name": "test1"}, {"name": "test2"}]}} + const result = await pb.collection("example").getOne("RECORD_ID", { + expand: "items", + fields: "name,expand.items.name", + }) + ``` + - Added new `./pocketbase admin` console command: ```sh // creates new admin account diff --git a/apis/base.go b/apis/base.go index 6e6b243e..0f4610ef 100644 --- a/apis/base.go +++ b/apis/base.go @@ -14,6 +14,7 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/echo/v5/middleware" "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/rest" "github.com/pocketbase/pocketbase/ui" "github.com/spf13/cast" ) @@ -25,6 +26,9 @@ const trailedAdminPath = "/_/" func InitApi(app core.App) (*echo.Echo, error) { e := echo.New() e.Debug = app.IsDebug() + e.JSONSerializer = &rest.Serializer{ + FieldsParam: "fields", + } // configure a custom router e.ResetRouterCreator(func(ec *echo.Echo) echo.Router { diff --git a/apis/record_helpers.go b/apis/record_helpers.go index a697515d..c86f4e12 100644 --- a/apis/record_helpers.go +++ b/apis/record_helpers.go @@ -121,7 +121,9 @@ func EnrichRecords(c echo.Context, dao *daos.Dao, records []*models.Record, defa } expands := defaultExpands - expands = append(expands, strings.Split(c.QueryParam(expandQueryParam), ",")...) + if param := c.QueryParam(expandQueryParam); param != "" { + expands = append(expands, strings.Split(param, ",")...) + } if len(expands) == 0 { return nil // nothing to expand } diff --git a/tools/rest/json_serializer.go b/tools/rest/json_serializer.go new file mode 100644 index 00000000..2e46423d --- /dev/null +++ b/tools/rest/json_serializer.go @@ -0,0 +1,121 @@ +package rest + +import ( + "encoding/json" + "strings" + + "github.com/labstack/echo/v5" + "github.com/pocketbase/pocketbase/tools/search" +) + +// Serializer represents custom REST JSON serializer based on echo.DefaultJSONSerializer, +// with support for additional generic response data transformation (eg. fields picker). +type Serializer struct { + echo.DefaultJSONSerializer + + FieldsParam string +} + +// Serialize converts an interface into a json and writes it to the response. +// +// It also provides a generic response data fields picker via the FieldsParam query parameter (default to "fields"). +func (s *Serializer) Serialize(c echo.Context, i any, indent string) error { + fieldsParam := s.FieldsParam + if fieldsParam == "" { + fieldsParam = "fields" + } + + param := c.QueryParam(fieldsParam) + if param == "" { + return s.DefaultJSONSerializer.Serialize(c, i, indent) + } + + fields := strings.Split(param, ",") + for i, f := range fields { + fields[i] = strings.TrimSpace(f) + } + + encoded, err := json.Marshal(i) + if err != nil { + return err + } + + var decoded any + + if err := json.Unmarshal(encoded, &decoded); err != nil { + return err + } + + var isSearchResult bool + + switch i.(type) { + case search.Result, *search.Result: + isSearchResult = true + } + + if isSearchResult { + if decodedMap, ok := decoded.(map[string]any); ok { + pickFields(decodedMap["items"], fields) + } + } else { + pickFields(decoded, fields) + } + + return s.DefaultJSONSerializer.Serialize(c, decoded, indent) +} + +func pickFields(data any, fields []string) { + switch v := data.(type) { + case map[string]any: + pickMapFields(v, fields) + case []map[string]any: + for _, item := range v { + pickMapFields(item, fields) + } + case []any: + if len(v) == 0 { + return // nothing to pick + } + + if _, ok := v[0].(map[string]any); !ok { + return // for now ignore non-map values + } + + for _, item := range v { + pickMapFields(item.(map[string]any), fields) + } + } +} + +func pickMapFields(data map[string]any, fields []string) { + if len(fields) == 0 { + return // nothing to pick + } + +DataLoop: + for k := range data { + matchingFields := make([]string, 0, len(fields)) + for _, f := range fields { + if strings.HasPrefix(f+".", k+".") { + matchingFields = append(matchingFields, f) + continue + } + } + + if len(matchingFields) == 0 { + delete(data, k) + continue DataLoop + } + + // trim the key from the fields + for i, v := range matchingFields { + trimmed := strings.TrimSuffix(strings.TrimPrefix(v+".", k+"."), ".") + if trimmed == "" { + continue DataLoop + } + matchingFields[i] = trimmed + } + + pickFields(data[k], matchingFields) + } +} diff --git a/tools/rest/json_serializer_test.go b/tools/rest/json_serializer_test.go new file mode 100644 index 00000000..3abb6f1c --- /dev/null +++ b/tools/rest/json_serializer_test.go @@ -0,0 +1,195 @@ +package rest_test + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/pocketbase/pocketbase/tools/rest" + "github.com/pocketbase/pocketbase/tools/search" +) + +func TestSerialize(t *testing.T) { + scenarios := []struct { + name string + serializer rest.Serializer + data any + query string + expected string + }{ + { + "empty query", + rest.Serializer{}, + map[string]any{"a": 1, "b": 2, "c": "test"}, + "", + `{"a":1,"b":2,"c":"test"}`, + }, + { + "empty fields", + rest.Serializer{}, + map[string]any{"a": 1, "b": 2, "c": "test"}, + "fields=", + `{"a":1,"b":2,"c":"test"}`, + }, + { + "missing fields", + rest.Serializer{}, + map[string]any{"a": 1, "b": 2, "c": "test"}, + "fields=missing", + `{}`, + }, + { + "non map response", + rest.Serializer{}, + "test", + "fields=a,b,test", + `"test"`, + }, + { + "non slice of map response", + rest.Serializer{}, + []any{"a", "b", "test"}, + "fields=a,test", + `["a","b","test"]`, + }, + { + "map with existing and missing fields", + rest.Serializer{}, + map[string]any{"a": 1, "b": 2, "c": "test"}, + "fields=a, c ,missing", // test individual fields trim + `{"a":1,"c":"test"}`, + }, + { + "custom fields param", + rest.Serializer{FieldsParam: "custom"}, + map[string]any{"a": 1, "b": 2, "c": "test"}, + "custom=a, c ,missing", // test individual fields trim + `{"a":1,"c":"test"}`, + }, + { + "slice of maps with existing and missing fields", + rest.Serializer{}, + []any{ + map[string]any{"a": 11, "b": 11, "c": "test1"}, + map[string]any{"a": 22, "b": 22, "c": "test2"}, + }, + "fields=a, c ,missing", // test individual fields trim + `[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`, + }, + { + "nested fields with mixed map and any slices", + rest.Serializer{}, + map[string]any{ + "a": 1, + "b": 2, + "c": "test", + "anySlice": []any{ + map[string]any{ + "A": []int{1, 2, 3}, + "B": []any{"1", "2", 3}, + "C": "test", + "D": map[string]any{ + "DA": 1, + "DB": 2, + }, + }, + map[string]any{ + "A": "test", + }, + }, + "mapSlice": []map[string]any{ + { + "A": []int{1, 2, 3}, + "B": []any{"1", "2", 3}, + "C": "test", + "D": []any{ + map[string]any{"DA": 1}, + }, + }, + { + "B": []any{"1", "2", 3}, + "D": []any{ + map[string]any{"DA": 2}, + map[string]any{"DA": 3}, + map[string]any{"DB": 4}, // will result to empty since there is no DA + }, + }, + }, + "fullMap": []map[string]any{ + { + "A": []int{1, 2, 3}, + "B": []any{"1", "2", 3}, + "C": "test", + }, + { + "B": []any{"1", "2", 3}, + "D": []any{ + map[string]any{"DA": 2}, + map[string]any{"DA": 3}, // will result to empty since there is no DA + }, + }, + }, + }, + "fields=a, c, anySlice.A, mapSlice.C, mapSlice.D.DA, anySlice.D,fullMap", + `{"a":1,"anySlice":[{"A":[1,2,3],"D":{"DA":1,"DB":2}},{"A":"test"}],"c":"test","fullMap":[{"A":[1,2,3],"B":["1","2",3],"C":"test"},{"B":["1","2",3],"D":[{"DA":2},{"DA":3}]}],"mapSlice":[{"C":"test","D":[{"DA":1}]},{"D":[{"DA":2},{"DA":3},{}]}]}`, + }, + { + "SearchResult", + rest.Serializer{}, + search.Result{ + Page: 1, + PerPage: 10, + TotalItems: 20, + TotalPages: 30, + Items: []any{ + map[string]any{"a": 11, "b": 11, "c": "test1"}, + map[string]any{"a": 22, "b": 22, "c": "test2"}, + }, + }, + "fields=a,c,missing", + `{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`, + }, + { + "*SearchResult", + rest.Serializer{}, + &search.Result{ + Page: 1, + PerPage: 10, + TotalItems: 20, + TotalPages: 30, + Items: []any{ + map[string]any{"a": 11, "b": 11, "c": "test1"}, + map[string]any{"a": 22, "b": 22, "c": "test2"}, + }, + }, + "fields=a,c", + `{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`, + }, + } + + for _, s := range scenarios { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.URL.RawQuery = s.query + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + if err := s.serializer.Serialize(c, s.data, ""); err != nil { + t.Errorf("[%s] Serialize failure: %v", s.name, err) + continue + } + + rawBody, err := io.ReadAll(rec.Result().Body) + if err != nil { + t.Errorf("[%s] Failed to read request body: %v", s.name, err) + continue + } + + if v := strings.TrimSpace(string(rawBody)); v != s.expected { + t.Fatalf("[%s] Expected body\n%v \ngot: \n%v", s.name, s.expected, v) + } + } +}