WebUI: add a JSON helper function

Signed-off-by: Nicola Murino <[email protected]>

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/cors v1.10.1
 	github.com/rs/xid v1.5.0
-	github.com/rs/zerolog v1.31.0
+	github.com/rs/zerolog v1.32.0
 	github.com/sftpgo/sdk v0.1.6-0.20240114195211-3f4916cc829c
 	github.com/shirou/gopsutil/v3 v3.24.1
 	github.com/spf13/afero v1.11.0

go.sum

@@ -338,8 +338,8 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
 github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
-github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
+github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=

internal/httpd/api_utils.go

@@ -299,6 +299,41 @@ func renderAPIDirContents(w http.ResponseWriter, r *http.Request, contents []os.
 	render.JSON(w, r, results)
 }
 
+func streamData(w io.Writer, data []byte) {
+	b := bytes.NewBuffer(data)
+	_, err := io.CopyN(w, b, int64(len(data)))
+	if err != nil {
+		panic(http.ErrAbortHandler)
+	}
+}
+
+func streamJSONArray(w http.ResponseWriter, chunkSize int, dataGetter func(limit, offset int) ([]byte, int, error)) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Accept-Ranges", "none")
+	w.WriteHeader(http.StatusOK)
+
+	streamData(w, []byte("["))
+	offset := 0
+	for {
+		data, count, err := dataGetter(chunkSize, offset)
+		if err != nil {
+			panic(http.ErrAbortHandler)
+		}
+		if count == 0 {
+			break
+		}
+		if offset > 0 {
+			streamData(w, []byte(","))
+		}
+		streamData(w, data[1:len(data)-1])
+		if count < chunkSize {
+			break
+		}
+		offset += count
+	}
+	streamData(w, []byte("]"))
+}
+
 func getCompressedFileName(username string, files []string) string {
 	if len(files) == 1 {
 		name := path.Base(files[0])

internal/httpd/httpd_test.go

@@ -7014,11 +7014,18 @@ func TestProviderErrors(t *testing.T) {
 	assert.Equal(t, http.StatusOK, rr.Code)
 	assert.Contains(t, rr.Body.String(), util.I18nErrorGetUser)
 
-	req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil)
-	assert.NoError(t, err)
-	setJWTCookieForReq(req, userWebToken)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
+	getJSONShares := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, err := http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil)
+		assert.NoError(t, err)
+		setJWTCookieForReq(req, userWebToken)
+		executeRequest(req)
+	}
+	getJSONShares()
+
 	req, err = http.NewRequest(http.MethodGet, webClientSharePath, nil)
 	assert.NoError(t, err)
 	setJWTCookieForReq(req, userWebToken)
@@ -7256,11 +7263,19 @@ func TestProviderErrors(t *testing.T) {
 	setJWTCookieForReq(req, testServerToken)
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusInternalServerError, rr)
-	req, err = http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil)
-	assert.NoError(t, err)
-	setJWTCookieForReq(req, testServerToken)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
+
+	getJSONActions := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, err := http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil)
+		assert.NoError(t, err)
+		setJWTCookieForReq(req, testServerToken)
+		executeRequest(req)
+	}
+	getJSONActions()
+
 	req, err = http.NewRequest(http.MethodGet, path.Join(webAdminEventRulePath, "rulename"), nil)
 	assert.NoError(t, err)
 	setJWTCookieForReq(req, testServerToken)
@@ -7271,11 +7286,19 @@ func TestProviderErrors(t *testing.T) {
 	setJWTCookieForReq(req, testServerToken)
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusInternalServerError, rr)
-	req, err = http.NewRequest(http.MethodGet, webAdminEventRulesPath+jsonAPISuffix+"?qlimit=10", nil)
-	assert.NoError(t, err)
-	setJWTCookieForReq(req, testServerToken)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
+
+	getJSONRules := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, err := http.NewRequest(http.MethodGet, webAdminEventRulesPath+jsonAPISuffix, nil)
+		assert.NoError(t, err)
+		setJWTCookieForReq(req, testServerToken)
+		executeRequest(req)
+	}
+	getJSONRules()
+
 	req, err = http.NewRequest(http.MethodGet, webAdminEventRulePath, nil)
 	assert.NoError(t, err)
 	setJWTCookieForReq(req, testServerToken)
@@ -18525,7 +18548,7 @@ func TestWebUserShare(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusOK, rr)
 
-	req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil) //nolint:goconst
+	req, err = http.NewRequest(http.MethodGet, webClientSharesPath+jsonAPISuffix, nil)
 	assert.NoError(t, err)
 	req.RemoteAddr = defaultRemoteAddr
 	setJWTCookieForReq(req, token)
@@ -22878,6 +22901,13 @@ func TestWebEventAction(t *testing.T) {
 	setCSRFHeaderForReq(req, csrfToken)
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusOK, rr)
+
+	req, err = http.NewRequest(http.MethodGet, webAdminEventActionsPath+jsonAPISuffix, nil)
+	assert.NoError(t, err)
+	setJWTCookieForReq(req, webToken)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr)
+	assert.Equal(t, `[]`, rr.Body.String())
 }
 
 func TestWebEventRule(t *testing.T) {
@@ -24928,18 +24958,39 @@ func TestProviderClosedMock(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusInternalServerError, rr)
 
-	req, _ = http.NewRequest(http.MethodGet, webFoldersPath+jsonAPISuffix, nil)
-	setJWTCookieForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
-	req, _ = http.NewRequest(http.MethodGet, webGroupsPath+jsonAPISuffix, nil)
-	setJWTCookieForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
-	req, _ = http.NewRequest(http.MethodGet, webUsersPath+jsonAPISuffix, nil)
-	setJWTCookieForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
+	getJSONFolders := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, _ := http.NewRequest(http.MethodGet, webFoldersPath+jsonAPISuffix, nil)
+		setJWTCookieForReq(req, token)
+		executeRequest(req)
+	}
+	getJSONFolders()
+
+	getJSONGroups := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, _ := http.NewRequest(http.MethodGet, webGroupsPath+jsonAPISuffix, nil)
+		setJWTCookieForReq(req, token)
+		executeRequest(req)
+	}
+	getJSONGroups()
+
+	getJSONUsers := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, _ := http.NewRequest(http.MethodGet, webUsersPath+jsonAPISuffix, nil)
+		setJWTCookieForReq(req, token)
+		executeRequest(req)
+	}
+	getJSONUsers()
+
 	req, _ = http.NewRequest(http.MethodGet, webUserPath+"/0", nil)
 	setJWTCookieForReq(req, token)
 	rr = executeRequest(req)
@@ -24961,10 +25012,16 @@ func TestProviderClosedMock(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusInternalServerError, rr)
 
-	req, _ = http.NewRequest(http.MethodGet, webAdminsPath+jsonAPISuffix, nil)
-	setJWTCookieForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
+	getJSONAdmins := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, _ := http.NewRequest(http.MethodGet, webAdminsPath+jsonAPISuffix, nil)
+		setJWTCookieForReq(req, token)
+		executeRequest(req)
+	}
+	getJSONAdmins()
 
 	req, _ = http.NewRequest(http.MethodGet, path.Join(webFolderPath, defaultTokenAuthUser), nil)
 	setJWTCookieForReq(req, token)
@@ -24998,11 +25055,17 @@ func TestProviderClosedMock(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusInternalServerError, rr)
 
-	req, err = http.NewRequest(http.MethodGet, webAdminRolesPath+jsonAPISuffix, nil)
-	assert.NoError(t, err)
-	setJWTCookieForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusInternalServerError, rr)
+	getJSONRoles := func() {
+		defer func() {
+			rcv := recover()
+			assert.Equal(t, http.ErrAbortHandler, rcv)
+		}()
+		req, err := http.NewRequest(http.MethodGet, webAdminRolesPath+jsonAPISuffix, nil)
+		assert.NoError(t, err)
+		setJWTCookieForReq(req, token)
+		executeRequest(req)
+	}
+	getJSONRoles()
 
 	req, err = http.NewRequest(http.MethodGet, path.Join(webAdminRolePath, role.Name), nil)
 	assert.NoError(t, err)

internal/httpd/internal_test.go

@@ -2195,6 +2195,33 @@ func TestRecoverer(t *testing.T) {
 	assert.Equal(t, http.StatusInternalServerError, rr.Code, rr.Body.String())
 }
 
+func TestStreamJSONArray(t *testing.T) {
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		return nil, 0, nil
+	}
+	rr := httptest.NewRecorder()
+	streamJSONArray(rr, 10, dataGetter)
+	assert.Equal(t, `[]`, rr.Body.String())
+
+	data := []int{}
+	for i := 0; i < 10; i++ {
+		data = append(data, i)
+	}
+
+	dataGetter = func(limit, offset int) ([]byte, int, error) {
+		if offset >= len(data) {
+			return nil, 0, nil
+		}
+		val := data[offset]
+		data, err := json.Marshal([]int{val})
+		return data, 1, err
+	}
+
+	rr = httptest.NewRecorder()
+	streamJSONArray(rr, 1, dataGetter)
+	assert.Equal(t, `[0,1,2,3,4,5,6,7,8,9]`, rr.Body.String())
+}
+
 func TestCompressorAbortHandler(t *testing.T) {
 	defer func() {
 		rcv := recover()
@@ -2209,6 +2236,15 @@ func TestCompressorAbortHandler(t *testing.T) {
 	renderCompressedFiles(&failingWriter{}, connection, "", nil, share)
 }
 
+func TestStreamDataAbortHandler(t *testing.T) {
+	defer func() {
+		rcv := recover()
+		assert.Equal(t, http.ErrAbortHandler, rcv)
+	}()
+
+	streamData(&failingWriter{}, []byte(`["a":"b"]`))
+}
+
 func TestZipErrors(t *testing.T) {
 	user := dataprovider.User{
 		BaseUser: sdk.BaseUser{

internal/httpd/webadmin.go

@@ -16,6 +16,7 @@ package httpd
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"html/template"
@@ -2878,19 +2879,17 @@ func getAllAdmins(w http.ResponseWriter, r *http.Request) {
 		sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden)
 		return
 	}
-	admins := make([]dataprovider.Admin, 0, 50)
-	for {
-		a, err := dataprovider.GetAdmins(defaultQueryLimit, len(admins), dataprovider.OrderASC)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetAdmins(limit, offset, dataprovider.OrderASC)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		admins = append(admins, a...)
-		if len(a) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, admins)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleGetWebAdmins(w http.ResponseWriter, r *http.Request) {
@@ -3043,19 +3042,17 @@ func getAllUsers(w http.ResponseWriter, r *http.Request) {
 		sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden)
 		return
 	}
-	users := make([]dataprovider.User, 0, 100)
-	for {
-		u, err := dataprovider.GetUsers(defaultQueryLimit, len(users), dataprovider.OrderASC, claims.Role)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetUsers(limit, offset, dataprovider.OrderASC, claims.Role)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		users = append(users, u...)
-		if len(u) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, users)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleGetWebUsers(w http.ResponseWriter, r *http.Request) {
@@ -3538,19 +3535,17 @@ func (s *httpdServer) getWebVirtualFolders(w http.ResponseWriter, r *http.Reques
 
 func getAllFolders(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	folders := make([]vfs.BaseVirtualFolder, 0, 50)
-	for {
-		f, err := dataprovider.GetFolders(defaultQueryLimit, len(folders), dataprovider.OrderASC, false)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetFolders(limit, offset, dataprovider.OrderASC, false)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		folders = append(folders, f...)
-		if len(f) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, folders)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleWebGetFolders(w http.ResponseWriter, r *http.Request) {
@@ -3578,19 +3573,17 @@ func (s *httpdServer) getWebGroups(w http.ResponseWriter, r *http.Request, limit
 
 func getAllGroups(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	groups := make([]dataprovider.Group, 0, 50)
-	for {
-		f, err := dataprovider.GetGroups(defaultQueryLimit, len(groups), dataprovider.OrderASC, false)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetGroups(limit, offset, dataprovider.OrderASC, false)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		groups = append(groups, f...)
-		if len(f) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, groups)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleWebGetGroups(w http.ResponseWriter, r *http.Request) {
@@ -3707,19 +3700,17 @@ func (s *httpdServer) getWebEventActions(w http.ResponseWriter, r *http.Request,
 
 func getAllActions(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	actions := make([]dataprovider.BaseEventAction, 0, 10)
-	for {
-		res, err := dataprovider.GetEventActions(defaultQueryLimit, len(actions), dataprovider.OrderASC, false)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetEventActions(limit, offset, dataprovider.OrderASC, false)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		actions = append(actions, res...)
-		if len(res) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, actions)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleWebGetEventActions(w http.ResponseWriter, r *http.Request) {
@@ -3819,19 +3810,17 @@ func (s *httpdServer) handleWebUpdateEventActionPost(w http.ResponseWriter, r *h
 
 func getAllRules(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	rules := make([]dataprovider.EventRule, 0, 10)
-	for {
-		res, err := dataprovider.GetEventRules(defaultQueryLimit, len(rules), dataprovider.OrderASC)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetEventRules(limit, offset, dataprovider.OrderASC)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		rules = append(rules, res...)
-		if len(res) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, rules)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleWebGetEventRules(w http.ResponseWriter, r *http.Request) {
@@ -3942,19 +3931,17 @@ func (s *httpdServer) getWebRoles(w http.ResponseWriter, r *http.Request, limit
 
 func getAllRoles(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	roles := make([]dataprovider.Role, 0, 10)
-	for {
-		res, err := dataprovider.GetRoles(defaultQueryLimit, len(roles), dataprovider.OrderASC, false)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		results, err := dataprovider.GetRoles(limit, offset, dataprovider.OrderASC, false)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		roles = append(roles, res...)
-		if len(res) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(results)
+		return data, len(results), err
 	}
-	render.JSON(w, r, roles)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleWebGetRoles(w http.ResponseWriter, r *http.Request) {

internal/httpd/webclient.go

@@ -1521,19 +1521,17 @@ func getAllShares(w http.ResponseWriter, r *http.Request) {
 		sendAPIResponse(w, r, nil, util.I18nErrorInvalidToken, http.StatusForbidden)
 		return
 	}
-	shares := make([]dataprovider.Share, 0, 10)
-	for {
-		sh, err := dataprovider.GetShares(defaultQueryLimit, len(shares), dataprovider.OrderASC, claims.Username)
+
+	dataGetter := func(limit, offset int) ([]byte, int, error) {
+		shares, err := dataprovider.GetShares(limit, offset, dataprovider.OrderASC, claims.Username)
 		if err != nil {
-			sendAPIResponse(w, r, err, getI18NErrorString(err, util.I18nError500Message), http.StatusInternalServerError)
-			return
-		}
-		shares = append(shares, sh...)
-		if len(sh) < defaultQueryLimit {
-			break
+			return nil, 0, err
 		}
+		data, err := json.Marshal(shares)
+		return data, len(shares), err
 	}
-	render.JSON(w, r, shares)
+
+	streamJSONArray(w, defaultQueryLimit, dataGetter)
 }
 
 func (s *httpdServer) handleClientGetShares(w http.ResponseWriter, r *http.Request) {