add some docs for telemetry server

move pprof to the telemetry server only

docs/full-configuration.md

@@ -160,6 +160,9 @@ The configuration file contains the following sections:
   - `auth_user_file`, string. Path to a file used to store usernames and passwords for basic authentication. This can be an absolute path or a path relative to the config dir. We support HTTP basic authentication, and the file format must conform to the one generated using the Apache `htpasswd` tool. The supported password formats are bcrypt (`$2y$` prefix) and md5 crypt (`$apr1$` prefix). If empty, HTTP authentication is disabled.
   - `certificate_file`, string. Certificate for HTTPS. This can be an absolute path or a path relative to the config dir.
   - `certificate_key_file`, string. Private key matching the above certificate. This can be an absolute path or a path relative to the config dir. If both the certificate and the private key are provided, the server will expect HTTPS connections. Certificate and key files can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows.
+- **"telemetry"**, the configuration for the telemetry server, more details [below](#telemetry-server)
+  - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 10000
+  - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"
 - **"http"**, the configuration for HTTP clients. HTTP clients are used for executing hooks such as the ones used for custom actions, external authentication and pre-login user modifications
   - `timeout`, integer. Timeout specifies a time limit, in seconds, for requests.
   - `ca_certificates`, list of strings. List of paths to extra CA certificates to trust. The paths can be absolute or relative to the config dir. Adding trusted CA certificates is a convenient way to use self-signed certificates without defeating the purpose of using TLS.
@@ -205,3 +208,11 @@ Let's see some examples:
 
 - To set sftpd `bind_port`, you need to define the env var `SFTPGO_SFTPD__BIND_PORT`
 - To set the `execute_on` actions, you need to define the env var `SFTPGO_COMMON__ACTIONS__EXECUTE_ON`. For example `SFTPGO_COMMON__ACTIONS__EXECUTE_ON=upload,download`
+
+## Telemetry Server
+
+The telemetry server exposes the following endpoints:
+
+- `/healthz`, health information (for health checks)
+- `/metrics`, Prometheus metrics
+- `/debug/pprof`, for pprof, more details [here](./profiling.md)

ftpd/ftpd_test.go

@@ -184,7 +184,7 @@ func TestMain(m *testing.M) {
 	}()
 
 	go func() {
-		if err := httpdConf.Initialize(configDir, false); err != nil {
+		if err := httpdConf.Initialize(configDir); err != nil {
 			logger.ErrorToConsole("could not start HTTP server: %v", err)
 			os.Exit(1)
 		}

httpd/httpd.go

@@ -39,7 +39,6 @@ const (
 	updateUsedQuotaPath       = "/api/v1/quota_update"
 	updateFolderUsedQuotaPath = "/api/v1/folder_quota_update"
 	metricsPath               = "/metrics"
-	pprofBasePath             = "/debug"
 	webBasePath               = "/web"
 	webUsersPath              = "/web/users"
 	webUserPath               = "/web/user"
@@ -101,7 +100,7 @@ type apiResponse struct {
 }
 
 // Initialize configures and starts the HTTP server
-func (c Conf) Initialize(configDir string, enableProfiler bool) error {
+func (c Conf) Initialize(configDir string) error {
 	var err error
 	logger.Debug(logSender, "", "initializing HTTP server with config %+v", c)
 	backupsPath = getConfigPath(c.BackupsPath, configDir)
@@ -127,7 +126,7 @@ func (c Conf) Initialize(configDir string, enableProfiler bool) error {
 	} else {
 		logger.Info(logSender, "", "built-in web interface disabled, please set templates_path and static_files_path to enable it")
 	}
-	initializeRouter(staticFilesPath, enableProfiler, enableWebAdmin)
+	initializeRouter(staticFilesPath, enableWebAdmin)
 	httpServer := &http.Server{
 		Addr:           fmt.Sprintf("%s:%d", c.BindAddress, c.BindPort),
 		Handler:        router,

httpd/httpd_test.go

@@ -171,7 +171,7 @@ func TestMain(m *testing.M) {
 	}
 
 	go func() {
-		if err := httpdConf.Initialize(configDir, true); err != nil {
+		if err := httpdConf.Initialize(configDir); err != nil {
 			logger.ErrorToConsole("could not start HTTP server: %v", err)
 			os.Exit(1)
 		}
@@ -196,7 +196,7 @@ func TestMain(m *testing.M) {
 	httpdConf.CertificateKeyFile = keyPath
 
 	go func() {
-		if err := httpdConf.Initialize(configDir, true); err != nil {
+		if err := httpdConf.Initialize(configDir); err != nil {
 			logger.ErrorToConsole("could not start HTTPS server: %v", err)
 			os.Exit(1)
 		}
@@ -223,31 +223,31 @@ func TestInitialization(t *testing.T) {
 	httpdConf := config.GetHTTPDConfig()
 	httpdConf.BackupsPath = "test_backups"
 	httpdConf.AuthUserFile = invalidFile
-	err = httpdConf.Initialize(configDir, true)
+	err = httpdConf.Initialize(configDir)
 	assert.Error(t, err)
 	httpdConf.BackupsPath = backupsPath
 	httpdConf.AuthUserFile = ""
 	httpdConf.CertificateFile = invalidFile
 	httpdConf.CertificateKeyFile = invalidFile
-	err = httpdConf.Initialize(configDir, true)
+	err = httpdConf.Initialize(configDir)
 	assert.Error(t, err)
 	httpdConf.CertificateFile = ""
 	httpdConf.CertificateKeyFile = ""
 	httpdConf.TemplatesPath = "."
-	err = httpdConf.Initialize(configDir, true)
+	err = httpdConf.Initialize(configDir)
 	assert.Error(t, err)
 	err = httpd.ReloadTLSCertificate()
 	assert.NoError(t, err, "reloading TLS Certificate must return nil error if no certificate is configured")
 	httpdConf = config.GetHTTPDConfig()
 	httpdConf.BackupsPath = ".."
-	err = httpdConf.Initialize(configDir, true)
+	err = httpdConf.Initialize(configDir)
 	assert.Error(t, err)
 	httpdConf.BackupsPath = backupsPath
 	httpdConf.CertificateFile = invalidFile
 	httpdConf.CertificateKeyFile = invalidFile
 	httpdConf.StaticFilesPath = ""
 	httpdConf.TemplatesPath = ""
-	err = httpdConf.Initialize(configDir, true)
+	err = httpdConf.Initialize(configDir)
 	assert.Error(t, err)
 }
 
@@ -2965,12 +2965,6 @@ func TestHealthCheck(t *testing.T) {
 	assert.Equal(t, "ok", rr.Body.String())
 }
 
-func TestPProfEndPointMock(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, pprofPath, nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr.Code)
-}
-
 func TestGetWebRootMock(t *testing.T) {
 	req, _ := http.NewRequest(http.MethodGet, "/", nil)
 	rr := executeRequest(req)

httpd/router.go

@@ -19,7 +19,7 @@ func GetHTTPRouter() http.Handler {
 	return router
 }
 
-func initializeRouter(staticFilesPath string, enableProfiler, enableWebAdmin bool) {
+func initializeRouter(staticFilesPath string, enableWebAdmin bool) {
 	router = chi.NewRouter()
 
 	router.Group(func(r chi.Router) {
@@ -34,12 +34,6 @@ func initializeRouter(staticFilesPath string, enableProfiler, enableWebAdmin boo
 		router.Use(logger.NewStructuredLogger(logger.GetLogger()))
 		router.Use(middleware.Recoverer)
 
-		if enableProfiler {
-			logger.InfoToConsole("enabling the built-in profiler")
-			logger.Info(logSender, "", "enabling the built-in profiler")
-			router.Mount(pprofBasePath, middleware.Profiler())
-		}
-
 		router.NotFound(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			sendAPIResponse(w, r, nil, "Not Found", http.StatusNotFound)
 		}))

service/service.go

@@ -146,7 +146,7 @@ func (s *Service) startServices() {
 
 	if httpdConf.BindPort > 0 {
 		go func() {
-			if err := httpdConf.Initialize(s.ConfigDir, s.Profiler); err != nil {
+			if err := httpdConf.Initialize(s.ConfigDir); err != nil {
 				logger.Error(logSender, "", "could not start HTTP server: %v", err)
 				logger.ErrorToConsole("could not start HTTP server: %v", err)
 				s.Error = err

sftpd/sftpd_test.go

@@ -244,7 +244,7 @@ func TestMain(m *testing.M) {
 	}()
 
 	go func() {
-		if err := httpdConf.Initialize(configDir, false); err != nil {
+		if err := httpdConf.Initialize(configDir); err != nil {
 			logger.ErrorToConsole("could not start HTTP server: %v", err)
 			os.Exit(1)
 		}

sftpgo.json

@@ -119,6 +119,10 @@
     "certificate_file": "",
     "certificate_key_file": ""
   },
+  "telemetry": {
+    "bind_port": 10000,
+    "bind_address": "127.0.0.1"
+  },
   "http": {
     "timeout": 20,
     "ca_certificates": [],

telemetry/telemetry.go

@@ -26,7 +26,7 @@ var (
 
 // Conf telemetry server configuration.
 type Conf struct {
-	// The port used for serving HTTP requests. 0 disable the HTTP server. Default: 8080
+	// The port used for serving HTTP requests. 0 disable the HTTP server. Default: 10000
 	BindPort int `json:"bind_port" mapstructure:"bind_port"`
 	// The address to listen on. A blank value means listen on all available network interfaces. Default: "127.0.0.1"
 	BindAddress string `json:"bind_address" mapstructure:"bind_address"`
@@ -42,7 +42,7 @@ func (c Conf) Initialize(enableProfiler bool) error {
 		ReadTimeout:    60 * time.Second,
 		WriteTimeout:   60 * time.Second,
 		IdleTimeout:    120 * time.Second,
-		MaxHeaderBytes: 1 << 16, // 64KB
+		MaxHeaderBytes: 1 << 14, // 16KB
 	}
 	return httpServer.ListenAndServe()
 }

webdavd/webdavd_test.go

@@ -182,7 +182,7 @@ func TestMain(m *testing.M) {
 	}()
 
 	go func() {
-		if err := httpdConf.Initialize(configDir, false); err != nil {
+		if err := httpdConf.Initialize(configDir); err != nil {
 			logger.ErrorToConsole("could not start HTTP server: %v", err)
 			os.Exit(1)
 		}