add HTTP hooks

external auth, pre-login user modification and keyboard interactive
authentication is now supported via HTTP requests too

config/config.go

@@ -54,15 +54,15 @@ func init() {
 				Command:             "",
 				HTTPNotificationURL: "",
 			},
-			Keys:                       []sftpd.Key{},
-			KexAlgorithms:              []string{},
-			Ciphers:                    []string{},
-			MACs:                       []string{},
-			LoginBannerFile:            "",
-			EnabledSSHCommands:         sftpd.GetDefaultSSHCommands(),
-			KeyboardInteractiveProgram: "",
-			ProxyProtocol:              0,
-			ProxyAllowed:               []string{},
+			Keys:                    []sftpd.Key{},
+			KexAlgorithms:           []string{},
+			Ciphers:                 []string{},
+			MACs:                    []string{},
+			LoginBannerFile:         "",
+			EnabledSSHCommands:      sftpd.GetDefaultSSHCommands(),
+			KeyboardInteractiveHook: "",
+			ProxyProtocol:           0,
+			ProxyAllowed:            []string{},
 		},
 		ProviderConf: dataprovider.Config{
 			Driver:           "sqlite",
@@ -83,10 +83,10 @@ func init() {
 				Command:             "",
 				HTTPNotificationURL: "",
 			},
-			ExternalAuthProgram: "",
-			ExternalAuthScope:   0,
-			CredentialsPath:     "credentials",
-			PreLoginProgram:     "",
+			ExternalAuthHook:  "",
+			ExternalAuthScope: 0,
+			CredentialsPath:   "credentials",
+			PreLoginHook:      "",
 		},
 		HTTPDConfig: httpd.Conf{
 			BindPort:           8080,
@@ -203,6 +203,25 @@ func LoadConfig(configDir, configName string) error {
 		logger.Warn(logSender, "", "Configuration error: %v", err)
 		logger.WarnToConsole("Configuration error: %v", err)
 	}
+	checkHooksCompatibility()
 	logger.Debug(logSender, "", "config file used: '%#v', config loaded: %+v", viper.ConfigFileUsed(), getRedactedGlobalConf())
 	return err
 }
+
+func checkHooksCompatibility() {
+	if len(globalConf.ProviderConf.ExternalAuthProgram) > 0 && len(globalConf.ProviderConf.ExternalAuthHook) == 0 {
+		logger.Warn(logSender, "", "external_auth_program is deprecated, please use external_auth_hook")
+		logger.WarnToConsole("external_auth_program is deprecated, please use external_auth_hook")
+		globalConf.ProviderConf.ExternalAuthHook = globalConf.ProviderConf.ExternalAuthProgram
+	}
+	if len(globalConf.ProviderConf.PreLoginProgram) > 0 && len(globalConf.ProviderConf.PreLoginHook) == 0 {
+		logger.Warn(logSender, "", "pre_login_program is deprecated, please use pre_login_hook")
+		logger.WarnToConsole("pre_login_program is deprecated, please use pre_login_hook")
+		globalConf.ProviderConf.PreLoginHook = globalConf.ProviderConf.PreLoginProgram
+	}
+	if len(globalConf.SFTPD.KeyboardInteractiveProgram) > 0 && len(globalConf.SFTPD.KeyboardInteractiveHook) == 0 {
+		logger.Warn(logSender, "", "keyboard_interactive_auth_program is deprecated, please use keyboard_interactive_auth_hook")
+		logger.WarnToConsole("keyboard_interactive_auth_program is deprecated, please use keyboard_interactive_auth_hook")
+		globalConf.SFTPD.KeyboardInteractiveHook = globalConf.SFTPD.KeyboardInteractiveProgram
+	}
+}

config/config_test.go

@@ -182,6 +182,48 @@ func TestInvalidUsersBaseDir(t *testing.T) {
 	os.Remove(configFilePath)
 }
 
+func TestHookCompatibity(t *testing.T) {
+	configDir := ".."
+	confName := tempConfigName + ".json"
+	configFilePath := filepath.Join(configDir, confName)
+	config.LoadConfig(configDir, "")
+	providerConf := config.GetProviderConf()
+	providerConf.ExternalAuthProgram = "ext_auth_program"
+	providerConf.PreLoginProgram = "pre_login_program"
+	c := make(map[string]dataprovider.Config)
+	c["data_provider"] = providerConf
+	jsonConf, _ := json.Marshal(c)
+	err := ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	if err != nil {
+		t.Errorf("error saving temporary configuration")
+	}
+	config.LoadConfig(configDir, tempConfigName)
+	providerConf = config.GetProviderConf()
+	if providerConf.ExternalAuthHook != "ext_auth_program" {
+		t.Error("unexpected external auth hook")
+	}
+	if providerConf.PreLoginHook != "pre_login_program" {
+		t.Error("unexpected pre-login hook")
+	}
+	os.Remove(configFilePath)
+
+	sftpdConf := config.GetSFTPDConfig()
+	sftpdConf.KeyboardInteractiveProgram = "key_int_program"
+	cnf := make(map[string]sftpd.Configuration)
+	cnf["sftpd"] = sftpdConf
+	jsonConf, _ = json.Marshal(cnf)
+	err = ioutil.WriteFile(configFilePath, jsonConf, 0666)
+	if err != nil {
+		t.Errorf("error saving temporary configuration")
+	}
+	config.LoadConfig(configDir, tempConfigName)
+	sftpdConf = config.GetSFTPDConfig()
+	if sftpdConf.KeyboardInteractiveHook != "key_int_program" {
+		t.Error("unexpected keyboard interactive hook")
+	}
+	os.Remove(configFilePath)
+}
+
 func TestSetGetConfig(t *testing.T) {
 	sftpdConf := config.GetSFTPDConfig()
 	sftpdConf.IdleTimeout = 3

dataprovider/bolt.go

@@ -53,7 +53,7 @@ type compatUserV2 struct {
 
 func initializeBoltProvider(basePath string) error {
 	var err error
-	logSender = BoltDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", BoltDataProviderName)
 	dbPath := config.Name
 	if !utils.IsFileInputValid(dbPath) {
 		return fmt.Errorf("Invalid database path: %#v", dbPath)

dataprovider/dataprovider.go

@@ -32,6 +32,8 @@ import (
 	"time"
 
 	"github.com/alexedwards/argon2id"
+	"github.com/go-chi/render"
+	"github.com/rs/xid"
 	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/crypto/pbkdf2"
 	"golang.org/x/crypto/ssh"
@@ -158,31 +160,46 @@ type Config struct {
 	// Actions to execute on user add, update, delete.
 	// Update action will not be fired for internal updates such as the last login or the user quota fields.
 	Actions Actions `json:"actions" mapstructure:"actions"`
-	// Absolute path to an external program to use for users authentication. Leave empty to use builtin
-	// authentication.
+	// Deprecated: please use ExternalAuthHook
+	ExternalAuthProgram string `json:"external_auth_program" mapstructure:"external_auth_program"`
+	// Absolute path to an external program or an HTTP URL to invoke for users authentication.
+	// Leave empty to use builtin authentication.
 	// The external program can read the following environment variables to get info about the user trying
 	// to authenticate:
 	//
 	// - SFTPGO_AUTHD_USERNAME
 	// - SFTPGO_AUTHD_PASSWORD, not empty for password authentication
 	// - SFTPGO_AUTHD_PUBLIC_KEY, not empty for public key authentication
+	// - SFTPGO_AUTHD_KEYBOARD_INTERACTIVE, not empty for keyboard interactive authentication
 	//
 	// The content of these variables is _not_ quoted. They may contain special characters. They are under the
 	// control of a possibly malicious remote user.
 	//
 	// The program must respond on the standard output with a valid SFTPGo user serialized as JSON if the
 	// authentication succeed or a user with an empty username if the authentication fails.
+	// If the hook is an HTTP URL then it will be invoked as HTTP POST.
+	// The request body will contain a JSON serialized struct with the following fields:
+	//
+	// - username
+	// - password, not empty for password authentication
+	// - public_key, not empty for public key authentication
+	// - keyboard_interactive, not empty for keyboard interactive authentication
+	//
+	// If authentication succeed the HTTP response code must be 200 and the response body a valid SFTPGo user
+	// serialized as JSON. If the authentication fails the HTTP response code must be != 200 or the response body
+	// must be empty.
+	//
 	// If the authentication succeed the user will be automatically added/updated inside the defined data provider.
 	// Actions defined for user added/updated will not be executed in this case.
-	// The external program should check authentication only, if there are login restrictions such as user
+	// The external hook should check authentication only, if there are login restrictions such as user
 	// disabled, expired, login allowed only from specific IP addresses it is enough to populate the matching user
 	// fields and these conditions will be checked in the same way as for builtin users.
-	// The external auth program must finish within 60 seconds.
+	// The external auth program must finish within 30 seconds.
 	// This method is slower than built-in authentication methods, but it's very flexible as anyone can
-	// easily write his own authentication programs.
-	ExternalAuthProgram string `json:"external_auth_program" mapstructure:"external_auth_program"`
-	// ExternalAuthScope defines the scope for the external authentication program.
-	// - 0 means all supported authetication scopes, the external program will be used for password,
+	// easily write his own authentication hooks.
+	ExternalAuthHook string `json:"external_auth_hook" mapstructure:"external_auth_hook"`
+	// ExternalAuthScope defines the scope for the external authentication hook.
+	// - 0 means all supported authetication scopes, the external hook will be executed for password,
 	//     public key and keyboard interactive authentication
 	// - 1 means passwords only
 	// - 2 means public keys only
@@ -194,11 +211,11 @@ type Config struct {
 	// Google Cloud Storage credentials. It can be a path relative to the config dir or an
 	// absolute path
 	CredentialsPath string `json:"credentials_path" mapstructure:"credentials_path"`
-	// Absolute path to an external program to start just before the user login.
-	// This program will be started before an existing user try to login and allows to
-	// modify or create the user.
-	// It is useful if you have users with dynamic fields that need to the updated just
-	// before the login.
+	// Deprecated: please use PreLoginHook
+	PreLoginProgram string `json:"pre_login_program" mapstructure:"pre_login_program"`
+	// Absolute path to an external program or an HTTP URL to invoke just before the user login.
+	// This program/URL allows to modify or create the user trying to login.
+	// It is useful if you have users with dynamic fields to update just before the login.
 	// The external program can read the following environment variables:
 	//
 	// - SFTPGO_LOGIND_USER, it contains the user trying to login serialized as JSON
@@ -206,20 +223,27 @@ type Config struct {
 	//
 	// The program must write on its standard output an empty string if no user update is needed
 	// or a valid SFTPGo user serialized as JSON.
+	//
+	// If the hook is an HTTP URL then it will be invoked as HTTP POST.
+	// The login method is added to the query string, for example "<http_url>?login_method=password".
+	// The request body will contain the user trying to login serialized as JSON.
+	// If no modification is needed the HTTP response code must be 204, otherwise the response code
+	// must be 200 and the response body a valid SFTPGo user serialized as JSON.
+	//
 	// The JSON response can include only the fields to update instead of the full user,
 	// for example if you want to disable the user you can return a response like this:
 	//
 	// {"status":0}
 	//
-	// Please note that if you want to create a new user, the pre-login program response must
+	// Please note that if you want to create a new user, the pre-login hook response must
 	// include all the mandatory user fields.
 	//
-	// The external program must finish within 60 seconds.
+	// The pre-login hook must finish within 30 seconds.
 	//
-	// If an error happens while executing the "PreLoginProgram" then login will be denied.
-	// PreLoginProgram and ExternalAuthProgram are mutally exclusive.
+	// If an error happens while executing the "PreLoginHook" then login will be denied.
+	// PreLoginHook and ExternalAuthHook are mutally exclusive.
 	// Leave empty to disable.
-	PreLoginProgram string `json:"pre_login_program" mapstructure:"pre_login_program"`
+	PreLoginHook string `json:"pre_login_hook" mapstructure:"pre_login_hook"`
 }
 
 // BackupData defines the structure for the backup/restore files
@@ -227,7 +251,15 @@ type BackupData struct {
 	Users []User `json:"users"`
 }
 
-type keyboardAuthProgramResponse struct {
+type keyboardAuthHookRequest struct {
+	RequestID string   `json:"request_id"`
+	Username  string   `json:"username,omitempty"`
+	Password  string   `json:"password,omitempty"`
+	Answers   []string `json:"answers,omitempty"`
+	Questions []string `json:"questions,omitempty"`
+}
+
+type keyboardAuthHookResponse struct {
 	Instruction string   `json:"instruction"`
 	Questions   []string `json:"questions"`
 	Echos       []bool   `json:"echos"`
@@ -308,25 +340,8 @@ func Initialize(cnf Config, basePath string) error {
 	config = cnf
 	sqlPlaceholders = getSQLPlaceholders()
 
-	if len(config.ExternalAuthProgram) > 0 {
-		if !filepath.IsAbs(config.ExternalAuthProgram) {
-			return fmt.Errorf("invalid external auth program: %#v must be an absolute path", config.ExternalAuthProgram)
-		}
-		_, err := os.Stat(config.ExternalAuthProgram)
-		if err != nil {
-			providerLog(logger.LevelWarn, "invalid external auth program: %v", err)
-			return err
-		}
-	}
-	if len(config.PreLoginProgram) > 0 {
-		if !filepath.IsAbs(config.PreLoginProgram) {
-			return fmt.Errorf("invalid pre-login program: %#v must be an absolute path", config.PreLoginProgram)
-		}
-		_, err := os.Stat(config.PreLoginProgram)
-		if err != nil {
-			providerLog(logger.LevelWarn, "invalid pre-login program: %v", err)
-			return err
-		}
+	if err = validateHooks(); err != nil {
+		return err
 	}
 	if err = validateCredentialsDir(basePath); err != nil {
 		return err
@@ -344,6 +359,30 @@ func Initialize(cnf Config, basePath string) error {
 	return nil
 }
 
+func validateHooks() error {
+	if len(config.PreLoginHook) > 0 && !strings.HasPrefix(config.PreLoginHook, "http") {
+		if !filepath.IsAbs(config.PreLoginHook) {
+			return fmt.Errorf("invalid pre-login hook: %#v must be an absolute path", config.PreLoginHook)
+		}
+		_, err := os.Stat(config.PreLoginHook)
+		if err != nil {
+			providerLog(logger.LevelWarn, "invalid pre-login hook: %v", err)
+			return err
+		}
+	}
+	if len(config.ExternalAuthHook) > 0 && !strings.HasPrefix(config.ExternalAuthHook, "http") {
+		if !filepath.IsAbs(config.ExternalAuthHook) {
+			return fmt.Errorf("invalid external auth hook: %#v must be an absolute path", config.ExternalAuthHook)
+		}
+		_, err := os.Stat(config.ExternalAuthHook)
+		if err != nil {
+			providerLog(logger.LevelWarn, "invalid external auth hook: %v", err)
+			return err
+		}
+	}
+	return nil
+}
+
 // InitializeDatabase creates the initial database structure
 func InitializeDatabase(cnf Config, basePath string) error {
 	config = cnf
@@ -361,15 +400,15 @@ func InitializeDatabase(cnf Config, basePath string) error {
 
 // CheckUserAndPass retrieves the SFTP user with the given username and password if a match is found or an error
 func CheckUserAndPass(p Provider, username string, password string) (User, error) {
-	if len(config.ExternalAuthProgram) > 0 && (config.ExternalAuthScope == 0 || config.ExternalAuthScope&1 != 0) {
+	if len(config.ExternalAuthHook) > 0 && (config.ExternalAuthScope == 0 || config.ExternalAuthScope&1 != 0) {
 		user, err := doExternalAuth(username, password, "", "")
 		if err != nil {
 			return user, err
 		}
 		return checkUserAndPass(user, password)
 	}
-	if len(config.PreLoginProgram) > 0 {
-		user, err := executePreLoginProgram(username, SSHLoginMethodPassword)
+	if len(config.PreLoginHook) > 0 {
+		user, err := executePreLoginHook(username, SSHLoginMethodPassword)
 		if err != nil {
 			return user, err
 		}
@@ -380,15 +419,15 @@ func CheckUserAndPass(p Provider, username string, password string) (User, error
 
 // CheckUserAndPubKey retrieves the SFTP user with the given username and public key if a match is found or an error
 func CheckUserAndPubKey(p Provider, username string, pubKey string) (User, string, error) {
-	if len(config.ExternalAuthProgram) > 0 && (config.ExternalAuthScope == 0 || config.ExternalAuthScope&2 != 0) {
+	if len(config.ExternalAuthHook) > 0 && (config.ExternalAuthScope == 0 || config.ExternalAuthScope&2 != 0) {
 		user, err := doExternalAuth(username, "", pubKey, "")
 		if err != nil {
 			return user, "", err
 		}
 		return checkUserAndPubKey(user, pubKey)
 	}
-	if len(config.PreLoginProgram) > 0 {
-		user, err := executePreLoginProgram(username, SSHLoginMethodPublicKey)
+	if len(config.PreLoginHook) > 0 {
+		user, err := executePreLoginHook(username, SSHLoginMethodPublicKey)
 		if err != nil {
 			return user, "", err
 		}
@@ -399,20 +438,20 @@ func CheckUserAndPubKey(p Provider, username string, pubKey string) (User, strin
 
 // CheckKeyboardInteractiveAuth checks the keyboard interactive authentication and returns
 // the authenticated user or an error
-func CheckKeyboardInteractiveAuth(p Provider, username, authProgram string, client ssh.KeyboardInteractiveChallenge) (User, error) {
+func CheckKeyboardInteractiveAuth(p Provider, username, authHook string, client ssh.KeyboardInteractiveChallenge) (User, error) {
 	var user User
 	var err error
-	if len(config.ExternalAuthProgram) > 0 && (config.ExternalAuthScope == 0 || config.ExternalAuthScope&4 != 0) {
+	if len(config.ExternalAuthHook) > 0 && (config.ExternalAuthScope == 0 || config.ExternalAuthScope&4 != 0) {
 		user, err = doExternalAuth(username, "", "", "1")
-	} else if len(config.PreLoginProgram) > 0 {
-		user, err = executePreLoginProgram(username, SSHLoginMethodKeyboardInteractive)
+	} else if len(config.PreLoginHook) > 0 {
+		user, err = executePreLoginHook(username, SSHLoginMethodKeyboardInteractive)
 	} else {
 		user, err = p.userExists(username)
 	}
 	if err != nil {
 		return user, err
 	}
-	return doKeyboardInteractiveAuth(user, authProgram, client)
+	return doKeyboardInteractiveAuth(user, authHook, client)
 }
 
 // UpdateLastLogin updates the last login fields for the given SFTP user
@@ -1077,28 +1116,115 @@ func terminateInteractiveAuthProgram(cmd *exec.Cmd, isFinished bool) {
 	cmd.Process.Kill()
 }
 
-func handleInteractiveQuestions(client ssh.KeyboardInteractiveChallenge, response keyboardAuthProgramResponse,
-	user User, stdin io.WriteCloser) error {
+func validateKeyboardAuthResponse(response keyboardAuthHookResponse) error {
+	if len(response.Questions) == 0 {
+		err := errors.New("interactive auth error: hook response does not contain questions")
+		providerLog(logger.LevelInfo, "%v", err)
+		return err
+	}
+	if len(response.Questions) != len(response.Echos) {
+		err := fmt.Errorf("interactive auth error, http hook response questions don't match echos: %v %v",
+			len(response.Questions), len(response.Echos))
+		providerLog(logger.LevelInfo, "%v", err)
+		return err
+	}
+	return nil
+}
+
+func sendKeyboardAuthHTTPReq(url *url.URL, request keyboardAuthHookRequest) (keyboardAuthHookResponse, error) {
+	var response keyboardAuthHookResponse
+	httpClient := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+	reqAsJSON, err := json.Marshal(request)
+	if err != nil {
+		providerLog(logger.LevelWarn, "error serializing keyboard interactive auth request: %v", err)
+		return response, err
+	}
+	resp, err := httpClient.Post(url.String(), "application/json", bytes.NewBuffer(reqAsJSON))
+	if err != nil {
+		providerLog(logger.LevelWarn, "error getting keyboard interactive auth hook HTTP response: %v", err)
+		return response, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return response, fmt.Errorf("wrong keyboard interactive auth http status code: %v, expected 200", resp.StatusCode)
+	}
+	err = render.DecodeJSON(resp.Body, &response)
+	return response, err
+}
+
+func executeKeyboardInteractiveHTTPHook(user User, authHook string, client ssh.KeyboardInteractiveChallenge) (int, error) {
+	authResult := 0
+	var url *url.URL
+	url, err := url.Parse(authHook)
+	if err != nil {
+		providerLog(logger.LevelWarn, "invalid url for keyboard interactive hook %#v, error: %v", authHook, err)
+		return authResult, err
+	}
+	requestID := xid.New().String()
+	req := keyboardAuthHookRequest{
+		Username:  user.Username,
+		Password:  user.Password,
+		RequestID: requestID,
+	}
+	var response keyboardAuthHookResponse
+	for {
+		response, err = sendKeyboardAuthHTTPReq(url, req)
+		if err != nil {
+			return authResult, err
+		}
+		if response.AuthResult != 0 {
+			return response.AuthResult, err
+		}
+		if err = validateKeyboardAuthResponse(response); err != nil {
+			return authResult, err
+		}
+		answers, err := getKeyboardInteractiveAnswers(client, response, user)
+		if err != nil {
+			return authResult, err
+		}
+		req = keyboardAuthHookRequest{
+			RequestID: requestID,
+			Username:  user.Username,
+			Password:  user.Password,
+			Answers:   answers,
+			Questions: response.Questions,
+		}
+	}
+}
+
+func getKeyboardInteractiveAnswers(client ssh.KeyboardInteractiveChallenge, response keyboardAuthHookResponse,
+	user User) ([]string, error) {
 	questions := response.Questions
 	answers, err := client(user.Username, response.Instruction, questions, response.Echos)
 	if err != nil {
 		providerLog(logger.LevelInfo, "error getting interactive auth client response: %v", err)
-		return err
+		return answers, err
 	}
 	if len(answers) != len(questions) {
 		err = fmt.Errorf("client answers does not match questions, expected: %v actual: %v", questions, answers)
 		providerLog(logger.LevelInfo, "keyboard interactive auth error: %v", err)
-		return err
+		return answers, err
 	}
 	if len(answers) == 1 && response.CheckPwd > 0 {
 		_, err = checkUserAndPass(user, answers[0])
-		providerLog(logger.LevelInfo, "interactive auth program requested password validation for user %#v, validation error: %v",
+		providerLog(logger.LevelInfo, "interactive auth hook requested password validation for user %#v, validation error: %v",
 			user.Username, err)
 		if err != nil {
-			return err
+			return answers, err
 		}
 		answers[0] = "OK"
 	}
+	return answers, err
+}
+
+func handleProgramInteractiveQuestions(client ssh.KeyboardInteractiveChallenge, response keyboardAuthHookResponse,
+	user User, stdin io.WriteCloser) error {
+	answers, err := getKeyboardInteractiveAnswers(client, response, user)
+	if err != nil {
+		return err
+	}
 	for _, answer := range answers {
 		if runtime.GOOS == "windows" {
 			answer += "\r"
@@ -1113,31 +1239,31 @@ func handleInteractiveQuestions(client ssh.KeyboardInteractiveChallenge, respons
 	return nil
 }
 
-func doKeyboardInteractiveAuth(user User, authProgram string, client ssh.KeyboardInteractiveChallenge) (User, error) {
+func executeKeyboardInteractiveProgram(user User, authHook string, client ssh.KeyboardInteractiveChallenge) (int, error) {
+	authResult := 0
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
-	cmd := exec.CommandContext(ctx, authProgram)
+	cmd := exec.CommandContext(ctx, authHook)
 	cmd.Env = append(os.Environ(),
 		fmt.Sprintf("SFTPGO_AUTHD_USERNAME=%v", user.Username),
 		fmt.Sprintf("SFTPGO_AUTHD_PASSWORD=%v", user.Password))
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
-		return user, err
+		return authResult, err
 	}
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
-		return user, err
+		return authResult, err
 	}
 	err = cmd.Start()
 	if err != nil {
-		return user, err
+		return authResult, err
 	}
 	var once sync.Once
 	scanner := bufio.NewScanner(stdout)
-	authResult := 0
 	for scanner.Scan() {
-		var response keyboardAuthProgramResponse
-		err := json.Unmarshal(scanner.Bytes(), &response)
+		var response keyboardAuthHookResponse
+		err = json.Unmarshal(scanner.Bytes(), &response)
 		if err != nil {
 			providerLog(logger.LevelInfo, "interactive auth error parsing response: %v", err)
 			once.Do(func() { terminateInteractiveAuthProgram(cmd, false) })
@@ -1147,19 +1273,12 @@ func doKeyboardInteractiveAuth(user User, authProgram string, client ssh.Keyboar
 			authResult = response.AuthResult
 			break
 		}
-		if len(response.Questions) == 0 {
-			providerLog(logger.LevelInfo, "interactive auth error: program response does not contain questions")
-			once.Do(func() { terminateInteractiveAuthProgram(cmd, false) })
-			break
-		}
-		if len(response.Questions) != len(response.Echos) {
-			providerLog(logger.LevelInfo, "interactive auth error, program response questions don't match echos: %v %v",
-				len(response.Questions), len(response.Echos))
+		if err = validateKeyboardAuthResponse(response); err != nil {
 			once.Do(func() { terminateInteractiveAuthProgram(cmd, false) })
 			break
 		}
 		go func() {
-			err := handleInteractiveQuestions(client, response, user, stdin)
+			err := handleProgramInteractiveQuestions(client, response, user, stdin)
 			if err != nil {
 				once.Do(func() { terminateInteractiveAuthProgram(cmd, false) })
 			}
@@ -1168,6 +1287,18 @@ func doKeyboardInteractiveAuth(user User, authProgram string, client ssh.Keyboar
 	stdin.Close()
 	once.Do(func() { terminateInteractiveAuthProgram(cmd, true) })
 	go cmd.Process.Wait()
+
+	return authResult, err
+}
+
+func doKeyboardInteractiveAuth(user User, authHook string, client ssh.KeyboardInteractiveChallenge) (User, error) {
+	var authResult int
+	var err error
+	if strings.HasPrefix(authHook, "http") {
+		authResult, err = executeKeyboardInteractiveHTTPHook(user, authHook, client)
+	} else {
+		authResult, err = executeKeyboardInteractiveProgram(user, authHook, client)
+	}
 	if authResult != 1 {
 		return user, fmt.Errorf("keyboard interactive auth failed, result: %v", authResult)
 	}
@@ -1178,7 +1309,47 @@ func doKeyboardInteractiveAuth(user User, authProgram string, client ssh.Keyboar
 	return user, nil
 }
 
-func executePreLoginProgram(username, loginMethod string) (User, error) {
+func getPreLoginHookResponse(loginMethod string, userAsJSON []byte) ([]byte, error) {
+	timeout := 30 * time.Second
+	if strings.HasPrefix(config.PreLoginHook, "http") {
+		var url *url.URL
+		var result []byte
+		url, err := url.Parse(config.PreLoginHook)
+		if err != nil {
+			providerLog(logger.LevelWarn, "invalid url for pre-login hook %#v, error: %v", config.PreLoginHook, err)
+			return result, err
+		}
+		q := url.Query()
+		q.Add("login_method", loginMethod)
+		url.RawQuery = q.Encode()
+		httpClient := &http.Client{
+			Timeout: timeout,
+		}
+		resp, err := httpClient.Post(url.String(), "application/json", bytes.NewBuffer(userAsJSON))
+		if err != nil {
+			providerLog(logger.LevelWarn, "error getting pre-login hook response: %v", err)
+			return result, err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode == http.StatusNoContent {
+			return result, nil
+		}
+		if resp.StatusCode != http.StatusOK {
+			return result, fmt.Errorf("wrong pre-login hook http status code: %v, expected 200", resp.StatusCode)
+		}
+		return ioutil.ReadAll(resp.Body)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, config.PreLoginHook)
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("SFTPGO_LOGIND_USER=%v", string(userAsJSON)),
+		fmt.Sprintf("SFTPGO_LOGIND_METHOD=%v", loginMethod),
+	)
+	return cmd.Output()
+}
+
+func executePreLoginHook(username, loginMethod string) (User, error) {
 	u, err := provider.userExists(username)
 	if err != nil {
 		if _, ok := err.(*RecordNotFoundError); !ok {
@@ -1193,19 +1364,12 @@ func executePreLoginProgram(username, loginMethod string) (User, error) {
 	if err != nil {
 		return u, err
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, config.PreLoginProgram)
-	cmd.Env = append(os.Environ(),
-		fmt.Sprintf("SFTPGO_LOGIND_USER=%v", string(userAsJSON)),
-		fmt.Sprintf("SFTPGO_LOGIND_METHOD=%v", loginMethod))
-	out, err := cmd.Output()
+	out, err := getPreLoginHookResponse(loginMethod, userAsJSON)
 	if err != nil {
-		return u, fmt.Errorf("Pre-login program error: %v", err)
+		return u, fmt.Errorf("Pre-login hook error: %v", err)
 	}
 	if len(strings.TrimSpace(string(out))) == 0 {
-		providerLog(logger.LevelDebug, "empty response from pre-login program, no modification requested for user %#v id: %v",
+		providerLog(logger.LevelDebug, "empty response from pre-login hook, no modification requested for user %#v id: %v",
 			username, u.ID)
 		if u.ID == 0 {
 			return u, &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", username)}
@@ -1220,7 +1384,7 @@ func executePreLoginProgram(username, loginMethod string) (User, error) {
 	userLastLogin := u.LastLogin
 	err = json.Unmarshal(out, &u)
 	if err != nil {
-		return u, fmt.Errorf("Invalid pre-login program response %#v, error: %v", string(out), err)
+		return u, fmt.Errorf("Invalid pre-login hook response %#v, error: %v", string(out), err)
 	}
 	u.ID = userID
 	u.UsedQuotaSize = userUsedQuotaSize
@@ -1235,14 +1399,57 @@ func executePreLoginProgram(username, loginMethod string) (User, error) {
 	if err != nil {
 		return u, err
 	}
-	providerLog(logger.LevelDebug, "user %#v added/updated from pre-login program response, id: %v", username, userID)
+	providerLog(logger.LevelDebug, "user %#v added/updated from pre-login hook response, id: %v", username, userID)
 	return provider.userExists(username)
 }
 
+func getExternalAuthResponse(username, password, pkey, keyboardInteractive string) ([]byte, error) {
+	timeout := 30 * time.Second
+	if strings.HasPrefix(config.ExternalAuthHook, "http") {
+		var url *url.URL
+		var result []byte
+		url, err := url.Parse(config.ExternalAuthHook)
+		if err != nil {
+			providerLog(logger.LevelWarn, "invalid url for external auth hook %#v, error: %v", config.ExternalAuthHook, err)
+			return result, err
+		}
+		httpClient := &http.Client{
+			Timeout: timeout,
+		}
+		authRequest := make(map[string]string)
+		authRequest["username"] = username
+		authRequest["password"] = password
+		authRequest["public_key"] = pkey
+		authRequest["keyboard_interactive"] = keyboardInteractive
+		authRequestAsJSON, err := json.Marshal(authRequest)
+		if err != nil {
+			providerLog(logger.LevelWarn, "error serializing external auth request: %v", err)
+			return result, err
+		}
+		resp, err := httpClient.Post(url.String(), "application/json", bytes.NewBuffer(authRequestAsJSON))
+		if err != nil {
+			providerLog(logger.LevelWarn, "error getting external auth hook HTTP response: %v", err)
+			return result, err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return result, fmt.Errorf("wrong external auth http status code: %v, expected 200", resp.StatusCode)
+		}
+		return ioutil.ReadAll(resp.Body)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, config.ExternalAuthHook)
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("SFTPGO_AUTHD_USERNAME=%v", username),
+		fmt.Sprintf("SFTPGO_AUTHD_PASSWORD=%v", password),
+		fmt.Sprintf("SFTPGO_AUTHD_PUBLIC_KEY=%v", pkey),
+		fmt.Sprintf("SFTPGO_AUTHD_KEYBOARD_INTERACTIVE=%v", keyboardInteractive))
+	return cmd.Output()
+}
+
 func doExternalAuth(username, password, pubKey, keyboardInteractive string) (User, error) {
 	var user User
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
 	pkey := ""
 	if len(pubKey) > 0 {
 		k, err := ssh.ParsePublicKey([]byte(pubKey))
@@ -1251,13 +1458,7 @@ func doExternalAuth(username, password, pubKey, keyboardInteractive string) (Use
 		}
 		pkey = string(ssh.MarshalAuthorizedKey(k))
 	}
-	cmd := exec.CommandContext(ctx, config.ExternalAuthProgram)
-	cmd.Env = append(os.Environ(),
-		fmt.Sprintf("SFTPGO_AUTHD_USERNAME=%v", username),
-		fmt.Sprintf("SFTPGO_AUTHD_PASSWORD=%v", password),
-		fmt.Sprintf("SFTPGO_AUTHD_PUBLIC_KEY=%v", pkey),
-		fmt.Sprintf("SFTPGO_AUTHD_KEYBOARD_INTERACTIVE=%v", keyboardInteractive))
-	out, err := cmd.Output()
+	out, err := getExternalAuthResponse(username, password, pkey, keyboardInteractive)
 	if err != nil {
 		return user, fmt.Errorf("External auth error: %v", err)
 	}

dataprovider/memory.go

@@ -38,6 +38,7 @@ type MemoryProvider struct {
 }
 
 func initializeMemoryProvider(basePath string) error {
+	logSender = fmt.Sprintf("dataprovider_%v", MemoryDataProviderName)
 	configFile := ""
 	if utils.IsFileInputValid(config.Name) {
 		configFile = config.Name

dataprovider/mysql.go

@@ -29,7 +29,7 @@ type MySQLProvider struct {
 
 func initializeMySQLProvider() error {
 	var err error
-	logSender = MySQLDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", MySQLDataProviderName)
 	dbHandle, err := sql.Open("mysql", getMySQLConnectionString(false))
 	if err == nil {
 		providerLog(logger.LevelDebug, "mysql database handle created, connection string: %#v, pool size: %v",

dataprovider/pgsql.go

@@ -27,7 +27,7 @@ type PGSQLProvider struct {
 
 func initializePGSQLProvider() error {
 	var err error
-	logSender = PGSQLDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", PGSQLDataProviderName)
 	dbHandle, err := sql.Open("postgres", getPGSQLConnectionString(false))
 	if err == nil {
 		providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v",

dataprovider/sqlite.go

@@ -30,7 +30,7 @@ type SQLiteProvider struct {
 func initializeSQLiteProvider(basePath string) error {
 	var err error
 	var connectionString string
-	logSender = SQLiteDataProviderName
+	logSender = fmt.Sprintf("dataprovider_%v", SQLiteDataProviderName)
 	if len(config.ConnectionString) == 0 {
 		dbPath := config.Name
 		if !utils.IsFileInputValid(dbPath) {

docs/dynamic-user-mod.md

@@ -1,7 +1,7 @@
 # Dynamic user creation or modification
 
-Dynamic user creation or modification is supported via an external program that can be executed just before the user login.
-To enable dynamic user modification, you must set the absolute path of your program using the `pre_login_program` key in your configuration file.
+Dynamic user creation or modification is supported via an external program or an HTTP URL that can be invoked just before the user login.
+To enable dynamic user modification, you must set the absolute path of your program or an HTTP URL using the `pre_login_hook` key in your configuration file.
 
 The external program can read the following environment variables to get info about the user trying to login:
 
@@ -13,6 +13,9 @@ The program must write, on its the standard output:
 - an empty string (or no response at all) if the user should not be created/updated
 - or the SFTPGo user, JSON serialized, if you want create or update the given user
 
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The login method is added to the query string, for example `<http_url>?login_method=password`.
+The request body will contain the user trying to login serialized as JSON. If no modification is needed the HTTP response code must be 204, otherwise the response code must be 200 and the response body a valid SFTPGo user serialized as JSON.
+
 Actions defined for user's updates will not be executed in this case.
 
 The JSON response can include only the fields to update instead of the full user. For example, if you want to disable the user, you can return a response like this:
@@ -21,11 +24,11 @@ The JSON response can include only the fields to update instead of the full user
 {"status": 0}
 ```
 
-Please note that if you want to create a new user, the pre-login program response must include all the mandatory user fields.
+Please note that if you want to create a new user, the pre-login hook response must include all the mandatory user fields.
 
-The external program must finish within 60 seconds.
+The hook must finish within 30 seconds.
 
-If an error happens while executing your program then login will be denied.
+If an error happens while executing the hook then login will be denied.
 
 "Dynamic user creation or modification" and "External Authentication" are mutally exclusive, they are quite similar, the difference is that "External Authentication" returns an already authenticated user while using "Dynamic users modification" you simply create or update a user. The authentication will be checked inside SFTPGo.
 In other words while using "External Authentication" the external program receives the credentials of the user trying to login (for example the clear text password) and it need to validate them. While using "Dynamic users modification" the pre-login program receives the user stored inside the dataprovider (it includes the hashed password if any) and it can modify it, after the modification SFTPGo will check the credentials of the user trying to login.

docs/external-auth.md

@@ -1,6 +1,6 @@
 # External Authentication
 
-To enable external authentication, you must set the absolute path of your authentication program using `external_auth_program` key in your configuration file.
+To enable external authentication, you must set the absolute path of your authentication program or an HTTP URL using the `external_auth_hook` key in your configuration file.
 
 The external program can read the following environment variables to get info about the user trying to authenticate:
 
@@ -11,13 +11,24 @@ The external program can read the following environment variables to get info ab
 
 Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
 The program must write, on its standard output, a valid SFTPGo user serialized as JSON if the authentication succeed or a user with an empty username if the authentication fails.
+
+If the hook is an HTTP URL then it will be invoked as HTTP POST. The request body will contain a JSON serialized struct with the following fields:
+
+- `username`
+- `password`, not empty for password authentication
+- `public_key`, not empty for public key authentication
+- `keyboard_interactive`, not empty for keyboard interactive authentication
+
+If authentication succeed the HTTP response code must be 200 and the response body a valid SFTPGo user serialized as JSON. If the authentication fails the HTTP response code must be != 200 or the response body must be empty.
+
 If the authentication succeeds, the user will be automatically added/updated inside the defined data provider. Actions defined for users added/updated will not be executed in this case.
-The external program should check authentication only. If there are login restrictions such as user disabled, expired, or login allowed only from specific IP addresses, it is enough to populate the matching user fields, and these conditions will be checked in the same way as for built-in users.
-The external auth program should finish very quickly. It will be killed if it does not exit within 60 seconds.
-This method is slower than built-in authentication, but it's very flexible as anyone can easily write his own authentication program.
-You can also restrict the authentication scope for the external program using the `external_auth_scope` configuration key:
+The external hook should check authentication only. If there are login restrictions such as user disabled, expired, or login allowed only from specific IP addresses, it is enough to populate the matching user fields, and these conditions will be checked in the same way as for built-in users.
+The hook must finish within 30 seconds.
+
+This method is slower than built-in authentication, but it's very flexible as anyone can easily write his own authentication hooks.
+You can also restrict the authentication scope for the hook using the `external_auth_scope` configuration key:
 
-- 0 means all supported authetication scopes. The external program will be used for password, public key and keyboard interactive authentication
+- 0 means all supported authetication scopes. The external hook will be used for password, public key and keyboard interactive authentication
 - 1 means passwords only
 - 2 means public keys only
 - 4 means keyboard interactive only
@@ -36,4 +47,4 @@ else
 fi
 ```
 
-If you have an external authentication program that could be useful for others too, please let us know and/or send a pull request.
+If you have an external authentication hook that could be useful for others too, please let us know and/or send a pull request.

docs/full-configuration.md

@@ -63,7 +63,8 @@ The configuration file contains the following sections:
     - `cd`, `pwd`. Some SFTP clients do not support the SFTP SSH_FXP_REALPATH packet type, so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` does nothing and `pwd` always returns the `/` path.
     - `git-receive-pack`, `git-upload-pack`, `git-upload-archive`. These commands enable support for Git repositories over SSH. They need to be installed and in your system's `PATH`. Git commands are not allowed inside virtual folders or inside directories with file extensions filters.
     - `rsync`. The `rsync` command needs to be installed and in your system's `PATH`. We cannot avoid that rsync creates symlinks, so if the user has the permission to create symlinks, we add the option `--safe-links` to the received rsync command if it is not already set. This should prevent creating symlinks that point outside the home dir. If the user cannot create symlinks, we add the option `--munge-links` if it is not already set. This should make symlinks unusable (but manually recoverable). The `rsync` command interacts with the filesystem directly and it is not aware of virtual folders and file extensions filters, so it will be automatically disabled for users with these features enabled.
-  - `keyboard_interactive_auth_program`, string. Absolute path to an external program to use for keyboard interactive authentication. See the "Keyboard Interactive Authentication" paragraph for more details.
+  - `keyboard_interactive_auth_program`, string. Deprecated, please use `keyboard_interactive_auth_hook`.
+  - `keyboard_interactive_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication. See the "Keyboard Interactive Authentication" paragraph for more details.
   - `proxy_protocol`, integer. Support for [HAProxy PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable the proxy protocol. It provides a convenient way to safely transport connection information such as a client's address across multiple layers of NAT or TCP proxies to get the real client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported. If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too. For example, for HAProxy, add `send-proxy` or `send-proxy-v2` to each server configuration line. The following modes are supported:
     - 0, disabled
     - 1, enabled. Proxy header will be used and requests without proxy header will be accepted
@@ -92,10 +93,12 @@ The configuration file contains the following sections:
     - `execute_on`, list of strings. Valid values are `add`, `update`, `delete`. `update` action will not be fired for internal updates such as the last login or the user quota fields.
     - `command`, string. Absolute path to the command to execute. Leave empty to disable.
     - `http_notification_url`, a valid URL. Leave empty to disable.
-  - `external_auth_program`, string. Absolute path to an external program to use for users authentication. See the "External Authentication" paragraph for more details. Leave empty to disable.
+  - `external_auth_program`, string. Deprecated, please use `external_auth_hook`.
+  - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See the "External Authentication" paragraph for more details. Leave empty to disable.
   - `external_auth_scope`, integer. 0 means all supported authetication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. The flags can be combined, for example 6 means public keys and keyboard interactive
   - `credentials_path`, string. It defines the directory for storing user provided credential files such as Google Cloud Storage credentials. This can be an absolute path or a path relative to the config dir
-  - `pre_login_program`, string. Absolute path to an external program to use to modify user details just before the login. See the "Dynamic user modification" paragraph for more details. Leave empty to disable.
+  - `pre_login_program`, string. Deprecated, please use `pre_login_hook`.
+  - `pre_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to modify user details just before the login. See the "Dynamic user modification" paragraph for more details. Leave empty to disable.
 - **"httpd"**, the configuration for the HTTP server used to serve REST API
   - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
   - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"

docs/keyboard-interactive.md

@@ -4,7 +4,7 @@ Keyboard interactive authentication is, in general, a series of questions asked
 This authentication method is typically used for multi-factor authentication.
 There are no restrictions on the number of questions asked on a particular authentication stage; there are also no restrictions on the number of stages involving different sets of questions.
 
-To enable keyboard interactive authentication, you must set the absolute path of your authentication program using the `keyboard_interactive_auth_program` key in your configuration file.
+To enable keyboard interactive authentication, you must set the absolute path of your authentication program or an HTTP URL using the  `keyboard_interactive_auth_hook` key in your configuration file.
 
 The external program can read the following environment variables to get info about the user trying to authenticate:
 
@@ -18,7 +18,7 @@ The program must write the questions on its standard output, in a single line, u
 - `instruction`, string. A short description to show to the user that is trying to authenticate. Can be empty or omitted
 - `questions`, list of questions to be asked to the user
 - `echos` list of boolean flags corresponding to the questions (so the lengths of both lists must be the same) and indicating whether user's reply for a particular question should be echoed on the screen while they are typing: true if it should be echoed, or false if it should be hidden.
-- `check_password` optional integer. Ask exactly one question and set this field to 1 if the expected answer is the user password and you want SFTPGo to check it for you. If the password is correct, the returned response to the program is `OK`. If the password is wrong, the program will be terminated and an authentication error will be returned to the user that is trying to authenticate.
+- `check_password` optional integer. Ask exactly one question and set this field to 1 if the expected answer is the user password and you want that SFTPGo checks it for you. If the password is correct, the returned response to the program is `OK`. If the password is wrong, the program will be terminated and an authentication error will be returned to the user that is trying to authenticate.
 - `auth_result`, integer. Set this field to 1 to indicate successful authentication. 0 is ignored. Any other value means authentication error. If this field is found and it is different from 0 then SFTPGo will not read any other questions from the external program, and it will finalize the authentication.
 
 SFTPGo writes the user answers to the program standard input, one per line, in the same order as the questions.
@@ -72,3 +72,93 @@ else
 fi
 ```
 
+If the hook is an HTTP URL then it will be invoked as HTTP POST multiple times for each login request.
+The request body will contain a JSON struct with the following fields:
+
+- `request_id`, string. Unique request identifier
+- `username`, string
+- `password`, string. This is the hashed password as stored inside the data provider
+- `answers`, list of string. It will be null for the first request
+- `questions`, list of string. It will contains the previous asked questions. It will be null for the first request
+
+The HTTP response code must be 200 and the body must contain the same JSON struct described for the program.
+
+Let's see a basic sample, the configured hook is `http://127.0.0.1:8000/keyIntHookPwd`, as soon as the user try to login, SFTPGo makes this HTTP POST request:
+
+```
+POST /keyIntHookPwd HTTP/1.1
+Host: 127.0.0.1:8000
+User-Agent: Go-http-client/1.1
+Content-Length: 189
+Content-Type: application/json
+Accept-Encoding: gzip
+
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA=="}
+```
+
+as you can see in this first requests `answers` and `questions` are null.
+
+Here is the response that instructs SFTPGo to ask for the user password and to check it:
+
+```
+HTTP/1.1 200 OK
+Date: Tue, 31 Mar 2020 21:15:24 GMT
+Server: WSGIServer/0.2 CPython/3.8.2
+Content-Type: application/json
+X-Frame-Options: SAMEORIGIN
+Content-Length: 143
+
+{"questions": ["Password: "], "check_password": 1, "instruction": "This is a sample for keyboard interactive authentication", "echos": [false]}
+```
+
+The user enters the correct password and so SFTPGo makes a new HTTP POST, please note that the `request_id` is the same of the previous request, this time the asked `questions` and the user's `answers` are not null:
+
+```
+POST /keyIntHookPwd HTTP/1.1
+Host: 127.0.0.1:8000
+User-Agent: Go-http-client/1.1
+Content-Length: 233
+Content-Type: application/json
+Accept-Encoding: gzip
+
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["OK"],"questions":["Password: "]}
+```
+
+Here is the HTTP response that istructs SFTPGo to ask for a new question:
+
+```
+HTTP/1.1 200 OK
+Date: Tue, 31 Mar 2020 21:15:27 GMT
+Server: WSGIServer/0.2 CPython/3.8.2
+Content-Type: application/json
+X-Frame-Options: SAMEORIGIN
+Content-Length: 66
+
+{"questions": ["Question2: "], "instruction": "", "echos": [true]}
+```
+
+As soon as the user answer to this question, SFTPGo will make a new HTTP POST request with the user's answers:
+
+```
+POST /keyIntHookPwd HTTP/1.1
+Host: 127.0.0.1:8000
+User-Agent: Go-http-client/1.1
+Content-Length: 239
+Content-Type: application/json
+Accept-Encoding: gzip
+
+{"request_id":"bq1r5r7cdrpd2qtn25ng","username":"a","password":"$pbkdf2-sha512$150000$ClOPkLNujMTL$XktKy0xuJsOfMYBz+f2bIyPTdbvDTSnJ1q+7+zp/HPq5Qojwp6kcpSIiVHiwvbi8P6HFXI/D3UJv9BLcnQFqPA==","answers":["answer2"],"questions":["Question2: "]}
+```
+
+Here is the final HTTP response that allows the user login:
+
+```
+HTTP/1.1 200 OK
+Date: Tue, 31 Mar 2020 21:15:29 GMT
+Server: WSGIServer/0.2 CPython/3.8.2
+Content-Type: application/json
+X-Frame-Options: SAMEORIGIN
+Content-Length: 18
+
+{"auth_result": 1}
+```

sftpd/server.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/drakkan/sftpgo/dataprovider"
@@ -98,9 +99,11 @@ type Configuration struct {
 	// The following SSH commands are enabled by default: "md5sum", "sha1sum", "cd", "pwd".
 	// "*" enables all supported SSH commands.
 	EnabledSSHCommands []string `json:"enabled_ssh_commands" mapstructure:"enabled_ssh_commands"`
-	// Absolute path to an external program to use for keyboard interactive authentication.
-	// Leave empty to disable this authentication mode.
+	// Deprecated: please use KeyboardInteractiveHook
 	KeyboardInteractiveProgram string `json:"keyboard_interactive_auth_program" mapstructure:"keyboard_interactive_auth_program"`
+	// Absolute path to an external program or an HTTP URL to invoke for keyboard interactive authentication.
+	// Leave empty to disable this authentication mode.
+	KeyboardInteractiveHook string `json:"keyboard_interactive_auth_hook" mapstructure:"keyboard_interactive_auth_hook"`
 	// Support for HAProxy PROXY protocol.
 	// If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable
 	// the proxy protocol. It provides a convenient way to safely transport connection information
@@ -297,21 +300,23 @@ func (c Configuration) configureLoginBanner(serverConfig *ssh.ServerConfig, conf
 }
 
 func (c Configuration) configureKeyboardInteractiveAuth(serverConfig *ssh.ServerConfig) {
-	if len(c.KeyboardInteractiveProgram) == 0 {
+	if len(c.KeyboardInteractiveHook) == 0 {
 		return
 	}
-	if !filepath.IsAbs(c.KeyboardInteractiveProgram) {
-		logger.WarnToConsole("invalid keyboard interactive authentication program: %#v must be an absolute path",
-			c.KeyboardInteractiveProgram)
-		logger.Warn(logSender, "", "invalid keyboard interactive authentication program: %#v must be an absolute path",
-			c.KeyboardInteractiveProgram)
-		return
-	}
-	_, err := os.Stat(c.KeyboardInteractiveProgram)
-	if err != nil {
-		logger.WarnToConsole("invalid keyboard interactive authentication program:: %v", err)
-		logger.Warn(logSender, "", "invalid keyboard interactive authentication program:: %v", err)
-		return
+	if !strings.HasPrefix(c.KeyboardInteractiveHook, "http") {
+		if !filepath.IsAbs(c.KeyboardInteractiveHook) {
+			logger.WarnToConsole("invalid keyboard interactive authentication program: %#v must be an absolute path",
+				c.KeyboardInteractiveHook)
+			logger.Warn(logSender, "", "invalid keyboard interactive authentication program: %#v must be an absolute path",
+				c.KeyboardInteractiveHook)
+			return
+		}
+		_, err := os.Stat(c.KeyboardInteractiveHook)
+		if err != nil {
+			logger.WarnToConsole("invalid keyboard interactive authentication program:: %v", err)
+			logger.Warn(logSender, "", "invalid keyboard interactive authentication program:: %v", err)
+			return
+		}
 	}
 	serverConfig.KeyboardInteractiveCallback = func(conn ssh.ConnMetadata, client ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
 		sp, err := c.validateKeyboardInteractiveCredentials(conn, client)
@@ -575,7 +580,7 @@ func (c Configuration) validateKeyboardInteractiveCredentials(conn ssh.ConnMetad
 
 	method := dataprovider.SSHLoginMethodKeyboardInteractive
 	metrics.AddLoginAttempt(method)
-	if user, err = dataprovider.CheckKeyboardInteractiveAuth(dataProvider, conn.User(), c.KeyboardInteractiveProgram, client); err == nil {
+	if user, err = dataprovider.CheckKeyboardInteractiveAuth(dataProvider, conn.User(), c.KeyboardInteractiveHook, client); err == nil {
 		sshPerm, err = loginUser(user, method, conn.RemoteAddr().String(), "")
 	}
 	if err != nil {

sftpd/sftpd.go

@@ -58,7 +58,6 @@ var (
 	mutex                sync.RWMutex
 	openConnections      map[string]Connection
 	activeTransfers      []*Transfer
-	idleConnectionTicker *time.Ticker
 	idleTimeout          time.Duration
 	activeQuotaScans     []ActiveQuotaScan
 	dataProvider         dataprovider.Provider
@@ -182,7 +181,6 @@ func (a *actionNotification) AsEnvVars() []string {
 
 func init() {
 	openConnections = make(map[string]Connection)
-	idleConnectionTicker = time.NewTicker(5 * time.Minute)
 }
 
 // GetDefaultSSHCommands returns the SSH commands enabled as default
@@ -373,8 +371,7 @@ func GetConnectionsStats() []ConnectionStatus {
 func startIdleTimer(maxIdleTime time.Duration) {
 	idleTimeout = maxIdleTime
 	go func() {
-		for t := range idleConnectionTicker.C {
-			logger.Debug(logSender, "", "idle connections check ticker %v", t)
+		for range time.Tick(5 * time.Minute) {
 			CheckIdleConnections()
 		}
 	}()
@@ -401,7 +398,6 @@ func CheckIdleConnections() {
 			c.Log(logger.LevelInfo, logSender, "close idle connection, idle time: %v, close error: %v", idleTime, err)
 		}
 	}
-	logger.Debug(logSender, "", "check idle connections ended")
 }
 
 func addConnection(c Connection) {

sftpd/sftpd_test.go

@@ -145,7 +145,7 @@ func TestMain(m *testing.M) {
 	}
 	keyIntAuthPath = filepath.Join(homeBasePath, "keyintauth.sh")
 	ioutil.WriteFile(keyIntAuthPath, getKeyboardInteractiveScriptContent([]string{"1", "2"}, 0, false, 1), 0755)
-	sftpdConf.KeyboardInteractiveProgram = keyIntAuthPath
+	sftpdConf.KeyboardInteractiveHook = keyIntAuthPath
 
 	scpPath, err = exec.LookPath("scp")
 	if err != nil {
@@ -250,12 +250,12 @@ func TestInitialization(t *testing.T) {
 	if err == nil {
 		t.Error("Inizialize must fail, a SFTP server should be already running")
 	}
-	sftpdConf.KeyboardInteractiveProgram = "invalid_file"
+	sftpdConf.KeyboardInteractiveHook = "invalid_file"
 	err = sftpdConf.Initialize(configDir)
 	if err == nil {
 		t.Error("Inizialize must fail, a SFTP server should be already running")
 	}
-	sftpdConf.KeyboardInteractiveProgram = filepath.Join(homeBasePath, "invalid_file")
+	sftpdConf.KeyboardInteractiveHook = filepath.Join(homeBasePath, "invalid_file")
 	err = sftpdConf.Initialize(configDir)
 	if err == nil {
 		t.Error("Inizialize must fail, a SFTP server should be already running")
@@ -1370,7 +1370,7 @@ func TestPreLoginScript(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(u, false), 0755)
-	providerConf.PreLoginProgram = preLoginPath
+	providerConf.PreLoginHook = preLoginPath
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
 		t.Errorf("error initializing data provider")
@@ -1432,7 +1432,7 @@ func TestPreLoginUserCreation(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(u, false), 0755)
-	providerConf.PreLoginProgram = preLoginPath
+	providerConf.PreLoginHook = preLoginPath
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
 		t.Errorf("error initializing data provider")
@@ -1491,7 +1491,7 @@ func TestLoginExternalAuthPwdAndPubKey(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, 0, false), 0755)
-	providerConf.ExternalAuthProgram = extAuthPath
+	providerConf.ExternalAuthHook = extAuthPath
 	providerConf.ExternalAuthScope = 0
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
@@ -1581,7 +1581,7 @@ func TestLoginExternalAuthPwd(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, 0, false), 0755)
-	providerConf.ExternalAuthProgram = extAuthPath
+	providerConf.ExternalAuthHook = extAuthPath
 	providerConf.ExternalAuthScope = 1
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
@@ -1649,7 +1649,7 @@ func TestLoginExternalAuthPubKey(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, 0, false), 0755)
-	providerConf.ExternalAuthProgram = extAuthPath
+	providerConf.ExternalAuthHook = extAuthPath
 	providerConf.ExternalAuthScope = 2
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
@@ -1717,7 +1717,7 @@ func TestLoginExternalAuthInteractive(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, 0, false), 0755)
-	providerConf.ExternalAuthProgram = extAuthPath
+	providerConf.ExternalAuthHook = extAuthPath
 	providerConf.ExternalAuthScope = 4
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
@@ -1786,7 +1786,7 @@ func TestLoginExternalAuthErrors(t *testing.T) {
 	config.LoadConfig(configDir, "")
 	providerConf := config.GetProviderConf()
 	ioutil.WriteFile(extAuthPath, getExtAuthScriptContent(u, 0, true), 0755)
-	providerConf.ExternalAuthProgram = extAuthPath
+	providerConf.ExternalAuthHook = extAuthPath
 	providerConf.ExternalAuthScope = 0
 	err := dataprovider.Initialize(providerConf, configDir)
 	if err != nil {

sftpgo.json

@@ -26,6 +26,7 @@
       "scp"
     ],
     "keyboard_interactive_auth_program": "",
+    "keyboard_interactive_auth_hook": "",
     "proxy_protocol": 0,
     "proxy_allowed": []
   },
@@ -49,8 +50,10 @@
       "http_notification_url": ""
     },
     "external_auth_program": "",
+    "external_auth_hook": "",
     "external_auth_scope": 0,
     "credentials_path": "credentials",
+    "pre_login_hook": "",
     "pre_login_program": ""
   },
   "httpd": {