[pocketbase#3790] added MaxSize json field option

CHANGELOG.md

@@ -80,6 +80,9 @@
 
 - Fixed graceful shutdown handling and speed up a little the app termination time.
 
+- Added `MaxSize` `json` field option to prevent storing large json data in the db ([#3790](https://github.com/pocketbase/pocketbase/issues/3790)).
+  _Existing `json` fields are updated with a system migration to have a ~5MB size limit (it can be adjusted from the Admin UI)._
+
 
 ## v0.20.0-rc3
 

apis/collection_test.go

@@ -1141,7 +1141,7 @@ func TestCollectionsImport(t *testing.T) {
 			},
 			ExpectedEvents: map[string]int{
 				"OnCollectionsBeforeImportRequest": 1,
-				"OnModelBeforeDelete":              2,
+				"OnModelBeforeDelete":              1,
 			},
 			AfterTestFunc: func(t *testing.T, app *tests.TestApp, res *http.Response) {
 				collections := []*models.Collection{}

daos/view.go

@@ -232,6 +232,9 @@ func defaultViewField(name string) *schema.SchemaField {
 	return &schema.SchemaField{
 		Name: name,
 		Type: schema.FieldTypeJson,
+		Options: &schema.JsonOptions{
+			MaxSize: 1, // the size doesn't matter in this case
+		},
 	}
 }
 

forms/collections_import_test.go

@@ -206,7 +206,7 @@ func TestCollectionsImportSubmit(t *testing.T) {
 			expectError:            true,
 			expectCollectionsCount: totalCollections,
 			expectEvents: map[string]int{
-				"OnModelBeforeDelete": 2,
+				"OnModelBeforeDelete": 1,
 			},
 		},
 		{

forms/validators/record_data.go

@@ -301,8 +301,14 @@ func (validator *RecordDataValidator) checkJsonValue(field *schema.SchemaField,
 	}
 
 	raw, _ := types.ParseJsonRaw(value)
-	rawStr := strings.TrimSpace(raw.String())
 
+	options, _ := field.Options.(*schema.JsonOptions)
+
+	if len(raw) > options.MaxSize {
+		return validation.NewError("validation_json_size_limit", fmt.Sprintf("The maximum allowed JSON size is %v bytes", options.MaxSize))
+	}
+
+	rawStr := strings.TrimSpace(raw.String())
 	if field.Required && list.ExistInSlice(rawStr, emptyJsonValues) {
 		return requiredErr
 	}

forms/validators/record_data_test.go

@@ -862,16 +862,25 @@ func TestRecordDataValidatorValidateJson(t *testing.T) {
 		&schema.SchemaField{
 			Name: "field1",
 			Type: schema.FieldTypeJson,
+			Options: &schema.JsonOptions{
+				MaxSize: 10,
+			},
 		},
 		&schema.SchemaField{
 			Name:     "field2",
 			Required: true,
 			Type:     schema.FieldTypeJson,
+			Options: &schema.JsonOptions{
+				MaxSize: 9999,
+			},
 		},
 		&schema.SchemaField{
 			Name:   "field3",
 			Unique: true,
 			Type:   schema.FieldTypeJson,
+			Options: &schema.JsonOptions{
+				MaxSize: 9999,
+			},
 		},
 	)
 	if err := app.Dao().SaveCollection(collection); err != nil {
@@ -938,6 +947,15 @@ func TestRecordDataValidatorValidateJson(t *testing.T) {
 			nil,
 			[]string{"field2"},
 		},
+		{
+			"(json) check MaxSize constraint",
+			map[string]any{
+				"field1": `"123456789"`, // max 10bytes
+				"field2": 123,
+			},
+			nil,
+			[]string{"field1"},
+		},
 		{
 			"(json) check json text invalid obj, array and number normalizations",
 			map[string]any{
@@ -969,9 +987,9 @@ func TestRecordDataValidatorValidateJson(t *testing.T) {
 		{
 			"(json) valid data - all fields with normalizations",
 			map[string]any{
-				"field1": []string{"a", "b", "c"},
+				"field1": `"12345678"`,
 				"field2": 123,
-				"field3": `"test"`,
+				"field3": []string{"a", "b", "c"},
 			},
 			nil,
 			[]string{},
@@ -1273,29 +1291,30 @@ func TestRecordDataValidatorValidateRelation(t *testing.T) {
 
 func checkValidatorErrors(t *testing.T, dao *daos.Dao, record *models.Record, scenarios []testDataFieldScenario) {
 	for i, s := range scenarios {
-		validator := validators.NewRecordDataValidator(dao, record, s.files)
-		result := validator.Validate(s.data)
-
-		prefix := fmt.Sprintf("%d", i)
-		if s.name != "" {
-			prefix = s.name
+		prefix := s.name
+		if prefix == "" {
+			prefix = fmt.Sprintf("%d", i)
 		}
 
-		// parse errors
-		errs, ok := result.(validation.Errors)
-		if !ok && result != nil {
-			t.Errorf("[%s] Failed to parse errors %v", prefix, result)
-			continue
-		}
+		t.Run(prefix, func(t *testing.T) {
+			validator := validators.NewRecordDataValidator(dao, record, s.files)
+			result := validator.Validate(s.data)
 
-		// check errors
-		if len(errs) > len(s.expectedErrors) {
-			t.Errorf("[%s] Expected error keys %v, got %v", prefix, s.expectedErrors, errs)
-		}
-		for _, k := range s.expectedErrors {
-			if _, ok := errs[k]; !ok {
-				t.Errorf("[%s] Missing expected error key %q in %v", prefix, k, errs)
+			// parse errors
+			errs, ok := result.(validation.Errors)
+			if !ok && result != nil {
+				t.Fatalf("Failed to parse errors %v", result)
 			}
-		}
+
+			// check errors
+			if len(errs) > len(s.expectedErrors) {
+				t.Fatalf("Expected error keys %v, got %v", s.expectedErrors, errs)
+			}
+			for _, k := range s.expectedErrors {
+				if _, ok := errs[k]; !ok {
+					t.Fatalf("Missing expected error key %q in %v", k, errs)
+				}
+			}
+		})
 	}
 }

migrations/1702134272_set_default_json_max_size.go

@@ -0,0 +1,51 @@
+package migrations
+
+import (
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/daos"
+	"github.com/pocketbase/pocketbase/models"
+	"github.com/pocketbase/pocketbase/models/schema"
+)
+
+// Update all collections with json fields to have a default MaxSize json field option.
+func init() {
+	AppMigrations.Register(func(db dbx.Builder) error {
+		dao := daos.New(db)
+
+		// note: update even the view collections to prevent
+		// unnecessary change detections during the automigrate
+		collections := []*models.Collection{}
+		if err := dao.CollectionQuery().All(&collections); err != nil {
+			return err
+		}
+
+		for _, collection := range collections {
+			var needSave bool
+
+			for _, f := range collection.Schema.Fields() {
+				if f.Type != schema.FieldTypeJson {
+					continue
+				}
+
+				options, _ := f.Options.(*schema.JsonOptions)
+				if options != nil {
+					options = &schema.JsonOptions{}
+				}
+				options.MaxSize = 5242880 // 5mb
+				f.Options = options
+				needSave = true
+			}
+
+			if !needSave {
+				continue
+			}
+
+			// save only the collection model without updating its records table
+			if err := dao.Save(collection); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}, nil)
+}

models/schema/schema_field.go

@@ -612,21 +612,24 @@ func (o SelectOptions) IsMultiple() bool {
 // -------------------------------------------------------------------
 
 type JsonOptions struct {
+	MaxSize int `form:"maxSize" json:"maxSize"`
 }
 
 func (o JsonOptions) Validate() error {
-	return nil
+	return validation.ValidateStruct(&o,
+		validation.Field(&o.MaxSize, validation.Required, validation.Min(1)),
+	)
 }
 
 // -------------------------------------------------------------------
 
 var _ MultiValuer = (*FileOptions)(nil)
 
 type FileOptions struct {
-	MaxSelect int      `form:"maxSelect" json:"maxSelect"`
-	MaxSize   int      `form:"maxSize" json:"maxSize"` // in bytes
 	MimeTypes []string `form:"mimeTypes" json:"mimeTypes"`
 	Thumbs    []string `form:"thumbs" json:"thumbs"`
+	MaxSelect int      `form:"maxSelect" json:"maxSelect"`
+	MaxSize   int      `form:"maxSize" json:"maxSize"`
 	Protected bool     `form:"protected" json:"protected"`
 }
 

models/schema/schema_field_test.go

@@ -520,12 +520,12 @@ func TestSchemaFieldInitOptions(t *testing.T) {
 		{
 			schema.SchemaField{Type: schema.FieldTypeJson},
 			false,
-			`{"system":false,"id":"","name":"","type":"json","required":false,"presentable":false,"unique":false,"options":{}}`,
+			`{"system":false,"id":"","name":"","type":"json","required":false,"presentable":false,"unique":false,"options":{"maxSize":0}}`,
 		},
 		{
 			schema.SchemaField{Type: schema.FieldTypeFile},
 			false,
-			`{"system":false,"id":"","name":"","type":"file","required":false,"presentable":false,"unique":false,"options":{"maxSelect":0,"maxSize":0,"mimeTypes":null,"thumbs":null,"protected":false}}`,
+			`{"system":false,"id":"","name":"","type":"file","required":false,"presentable":false,"unique":false,"options":{"mimeTypes":null,"thumbs":null,"maxSelect":0,"maxSize":0,"protected":false}}`,
 		},
 		{
 			schema.SchemaField{Type: schema.FieldTypeRelation},
@@ -548,16 +548,18 @@ func TestSchemaFieldInitOptions(t *testing.T) {
 	}
 
 	for i, s := range scenarios {
-		err := s.field.InitOptions()
+		t.Run(fmt.Sprintf("s%d_%s", i, s.field.Type), func(t *testing.T) {
+			err := s.field.InitOptions()
 
-		hasErr := err != nil
-		if hasErr != s.expectError {
-			t.Errorf("(%d) Expected %v, got %v (%v)", i, s.expectError, hasErr, err)
-		}
+			hasErr := err != nil
+			if hasErr != s.expectError {
+				t.Fatalf("Expected %v, got %v (%v)", s.expectError, hasErr, err)
+			}
 
-		if s.field.String() != s.expectJson {
-			t.Errorf("(%d), Expected %v, got %v", i, s.expectJson, s.field.String())
-		}
+			if s.field.String() != s.expectJson {
+				t.Fatalf(" Expected\n%v\ngot\n%v", s.expectJson, s.field.String())
+			}
+		})
 	}
 }
 
@@ -2058,6 +2060,16 @@ func TestJsonOptionsValidate(t *testing.T) {
 		{
 			"empty",
 			schema.JsonOptions{},
+			[]string{"maxSize"},
+		},
+		{
+			"MaxSize < 0",
+			schema.JsonOptions{MaxSize: -1},
+			[]string{"maxSize"},
+		},
+		{
+			"MaxSize > 0",
+			schema.JsonOptions{MaxSize: 1},
 			[]string{},
 		},
 	}

plugins/migratecmd/migratecmd_test.go

@@ -167,11 +167,11 @@ func init() {
 
 			files, err := os.ReadDir(migrationsDir)
 			if err != nil {
-				t.Fatalf("Expected migrationsDir to be created, got: %v", err)
+				t.Fatalf("Expected migrationsDir to be created, got %v", err)
 			}
 
 			if total := len(files); total != 1 {
-				t.Fatalf("Expected 1 file to be generated, got %d", total)
+				t.Fatalf("Expected 1 file to be generated, got %d: %v", total, files)
 			}
 
 			expectedName := "_created_new_name." + s.lang

ui/dist/assets/AuthRefreshDocs-72e74bf1.js → ui/dist/assets/AuthRefreshDocs-a828e8e8.js

@@ -1,4 +1,4 @@
-import{S as je,i as xe,s as Je,N as Ue,O as J,e as s,w as k,b as p,c as K,f as b,g as d,h as o,m as I,x as de,P as Ee,Q as Ke,k as Ie,R as We,n as Ge,t as N,a as V,o as u,d as W,C as Le,p as Xe,r as G,u as Ye}from"./index-acadfc6c.js";import{S as Ze}from"./SdkTabs-164e71a2.js";import{F as et}from"./FieldsQueryParam-d76d2b80.js";function Ne(r,l,a){const n=r.slice();return n[5]=l[a],n}function Ve(r,l,a){const n=r.slice();return n[5]=l[a],n}function ze(r,l){let a,n=l[5].code+"",m,_,i,h;function g(){return l[4](l[5])}return{key:r,first:null,c(){a=s("button"),m=k(n),_=p(),b(a,"class","tab-item"),G(a,"active",l[1]===l[5].code),this.first=a},m(v,w){d(v,a,w),o(a,m),o(a,_),i||(h=Ye(a,"click",g),i=!0)},p(v,w){l=v,w&4&&n!==(n=l[5].code+"")&&de(m,n),w&6&&G(a,"active",l[1]===l[5].code)},d(v){v&&u(a),i=!1,h()}}}function Qe(r,l){let a,n,m,_;return n=new Ue({props:{content:l[5].body}}),{key:r,first:null,c(){a=s("div"),K(n.$$.fragment),m=p(),b(a,"class","tab-item"),G(a,"active",l[1]===l[5].code),this.first=a},m(i,h){d(i,a,h),I(n,a,null),o(a,m),_=!0},p(i,h){l=i;const g={};h&4&&(g.content=l[5].body),n.$set(g),(!_||h&6)&&G(a,"active",l[1]===l[5].code)},i(i){_||(N(n.$$.fragment,i),_=!0)},o(i){V(n.$$.fragment,i),_=!1},d(i){i&&u(a),W(n)}}}function tt(r){var De,Fe;let l,a,n=r[0].name+"",m,_,i,h,g,v,w,M,X,S,z,ue,Q,q,pe,Y,U=r[0].name+"",Z,he,fe,j,ee,D,te,T,oe,be,F,C,le,me,ae,_e,f,ke,R,ge,ve,$e,se,ye,ne,Se,we,Te,re,Ce,Pe,A,ie,O,ce,P,H,y=[],Re=new Map,Ae,E,$=[],Be=new Map,B;v=new Ze({props:{js:`
+import{S as je,i as xe,s as Je,N as Ue,O as J,e as s,w as k,b as p,c as K,f as b,g as d,h as o,m as I,x as de,P as Ee,Q as Ke,k as Ie,R as We,n as Ge,t as N,a as V,o as u,d as W,C as Le,p as Xe,r as G,u as Ye}from"./index-d606279c.js";import{S as Ze}from"./SdkTabs-38f5bb15.js";import{F as et}from"./FieldsQueryParam-558f4709.js";function Ne(r,l,a){const n=r.slice();return n[5]=l[a],n}function Ve(r,l,a){const n=r.slice();return n[5]=l[a],n}function ze(r,l){let a,n=l[5].code+"",m,_,i,h;function g(){return l[4](l[5])}return{key:r,first:null,c(){a=s("button"),m=k(n),_=p(),b(a,"class","tab-item"),G(a,"active",l[1]===l[5].code),this.first=a},m(v,w){d(v,a,w),o(a,m),o(a,_),i||(h=Ye(a,"click",g),i=!0)},p(v,w){l=v,w&4&&n!==(n=l[5].code+"")&&de(m,n),w&6&&G(a,"active",l[1]===l[5].code)},d(v){v&&u(a),i=!1,h()}}}function Qe(r,l){let a,n,m,_;return n=new Ue({props:{content:l[5].body}}),{key:r,first:null,c(){a=s("div"),K(n.$$.fragment),m=p(),b(a,"class","tab-item"),G(a,"active",l[1]===l[5].code),this.first=a},m(i,h){d(i,a,h),I(n,a,null),o(a,m),_=!0},p(i,h){l=i;const g={};h&4&&(g.content=l[5].body),n.$set(g),(!_||h&6)&&G(a,"active",l[1]===l[5].code)},i(i){_||(N(n.$$.fragment,i),_=!0)},o(i){V(n.$$.fragment,i),_=!1},d(i){i&&u(a),W(n)}}}function tt(r){var De,Fe;let l,a,n=r[0].name+"",m,_,i,h,g,v,w,M,X,S,z,ue,Q,q,pe,Y,U=r[0].name+"",Z,he,fe,j,ee,D,te,T,oe,be,F,C,le,me,ae,_e,f,ke,R,ge,ve,$e,se,ye,ne,Se,we,Te,re,Ce,Pe,A,ie,O,ce,P,H,y=[],Re=new Map,Ae,E,$=[],Be=new Map,B;v=new Ze({props:{js:`
         import PocketBase from 'pocketbase';
 
         const pb = new PocketBase('${r[3]}');

ui/dist/assets/AuthWithOAuth2Docs-03a42751.js → ui/dist/assets/AuthWithOAuth2Docs-892682be.js

@@ -1,4 +1,4 @@
-import{S as Ee,i as Je,s as Ne,N as Le,O as z,e as o,w as k,b as h,c as I,f as p,g as r,h as a,m as K,x as pe,P as Ue,Q as Qe,k as xe,R as ze,n as Ie,t as L,a as E,o as c,d as G,C as Be,p as Ke,r as X,u as Ge}from"./index-acadfc6c.js";import{S as Xe}from"./SdkTabs-164e71a2.js";import{F as Ye}from"./FieldsQueryParam-d76d2b80.js";function Fe(s,l,n){const i=s.slice();return i[5]=l[n],i}function He(s,l,n){const i=s.slice();return i[5]=l[n],i}function je(s,l){let n,i=l[5].code+"",f,g,d,b;function _(){return l[4](l[5])}return{key:s,first:null,c(){n=o("button"),f=k(i),g=h(),p(n,"class","tab-item"),X(n,"active",l[1]===l[5].code),this.first=n},m(v,O){r(v,n,O),a(n,f),a(n,g),d||(b=Ge(n,"click",_),d=!0)},p(v,O){l=v,O&4&&i!==(i=l[5].code+"")&&pe(f,i),O&6&&X(n,"active",l[1]===l[5].code)},d(v){v&&c(n),d=!1,b()}}}function Ve(s,l){let n,i,f,g;return i=new Le({props:{content:l[5].body}}),{key:s,first:null,c(){n=o("div"),I(i.$$.fragment),f=h(),p(n,"class","tab-item"),X(n,"active",l[1]===l[5].code),this.first=n},m(d,b){r(d,n,b),K(i,n,null),a(n,f),g=!0},p(d,b){l=d;const _={};b&4&&(_.content=l[5].body),i.$set(_),(!g||b&6)&&X(n,"active",l[1]===l[5].code)},i(d){g||(L(i.$$.fragment,d),g=!0)},o(d){E(i.$$.fragment,d),g=!1},d(d){d&&c(n),G(i)}}}function Ze(s){let l,n,i=s[0].name+"",f,g,d,b,_,v,O,P,Y,A,J,be,N,R,me,Z,Q=s[0].name+"",ee,fe,te,M,ae,W,le,U,ne,S,oe,ge,B,y,se,ke,ie,_e,m,ve,C,we,$e,Oe,re,Ae,ce,Se,ye,Te,de,Ce,qe,q,ue,F,he,T,H,$=[],De=new Map,Pe,j,w=[],Re=new Map,D;v=new Xe({props:{js:`
+import{S as Ee,i as Je,s as Ne,N as Le,O as z,e as o,w as k,b as h,c as I,f as p,g as r,h as a,m as K,x as pe,P as Ue,Q as Qe,k as xe,R as ze,n as Ie,t as L,a as E,o as c,d as G,C as Be,p as Ke,r as X,u as Ge}from"./index-d606279c.js";import{S as Xe}from"./SdkTabs-38f5bb15.js";import{F as Ye}from"./FieldsQueryParam-558f4709.js";function Fe(s,l,n){const i=s.slice();return i[5]=l[n],i}function He(s,l,n){const i=s.slice();return i[5]=l[n],i}function je(s,l){let n,i=l[5].code+"",f,g,d,b;function _(){return l[4](l[5])}return{key:s,first:null,c(){n=o("button"),f=k(i),g=h(),p(n,"class","tab-item"),X(n,"active",l[1]===l[5].code),this.first=n},m(v,O){r(v,n,O),a(n,f),a(n,g),d||(b=Ge(n,"click",_),d=!0)},p(v,O){l=v,O&4&&i!==(i=l[5].code+"")&&pe(f,i),O&6&&X(n,"active",l[1]===l[5].code)},d(v){v&&c(n),d=!1,b()}}}function Ve(s,l){let n,i,f,g;return i=new Le({props:{content:l[5].body}}),{key:s,first:null,c(){n=o("div"),I(i.$$.fragment),f=h(),p(n,"class","tab-item"),X(n,"active",l[1]===l[5].code),this.first=n},m(d,b){r(d,n,b),K(i,n,null),a(n,f),g=!0},p(d,b){l=d;const _={};b&4&&(_.content=l[5].body),i.$set(_),(!g||b&6)&&X(n,"active",l[1]===l[5].code)},i(d){g||(L(i.$$.fragment,d),g=!0)},o(d){E(i.$$.fragment,d),g=!1},d(d){d&&c(n),G(i)}}}function Ze(s){let l,n,i=s[0].name+"",f,g,d,b,_,v,O,P,Y,A,J,be,N,R,me,Z,Q=s[0].name+"",ee,fe,te,M,ae,W,le,U,ne,S,oe,ge,B,y,se,ke,ie,_e,m,ve,C,we,$e,Oe,re,Ae,ce,Se,ye,Te,de,Ce,qe,q,ue,F,he,T,H,$=[],De=new Map,Pe,j,w=[],Re=new Map,D;v=new Xe({props:{js:`
         import PocketBase from 'pocketbase';
 
         const pb = new PocketBase('${s[3]}');