common.go 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401
  1. // Copyright (C) 2019 Nicola Murino
  2. //
  3. // This program is free software: you can redistribute it and/or modify
  4. // it under the terms of the GNU Affero General Public License as published
  5. // by the Free Software Foundation, version 3.
  6. //
  7. // This program is distributed in the hope that it will be useful,
  8. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. // GNU Affero General Public License for more details.
  11. //
  12. // You should have received a copy of the GNU Affero General Public License
  13. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  14. // Package common defines code shared among file transfer packages and protocols
  15. package common
  16. import (
  17. "context"
  18. "errors"
  19. "fmt"
  20. "net"
  21. "net/http"
  22. "net/url"
  23. "os"
  24. "os/exec"
  25. "path/filepath"
  26. "strconv"
  27. "strings"
  28. "sync"
  29. "sync/atomic"
  30. "time"
  31. "github.com/pires/go-proxyproto"
  32. "github.com/sftpgo/sdk/plugin/notifier"
  33. "github.com/drakkan/sftpgo/v2/internal/command"
  34. "github.com/drakkan/sftpgo/v2/internal/dataprovider"
  35. "github.com/drakkan/sftpgo/v2/internal/httpclient"
  36. "github.com/drakkan/sftpgo/v2/internal/logger"
  37. "github.com/drakkan/sftpgo/v2/internal/metric"
  38. "github.com/drakkan/sftpgo/v2/internal/plugin"
  39. "github.com/drakkan/sftpgo/v2/internal/smtp"
  40. "github.com/drakkan/sftpgo/v2/internal/util"
  41. "github.com/drakkan/sftpgo/v2/internal/vfs"
  42. )
  43. // constants
  44. const (
  45. logSender = "common"
  46. uploadLogSender = "Upload"
  47. downloadLogSender = "Download"
  48. renameLogSender = "Rename"
  49. rmdirLogSender = "Rmdir"
  50. mkdirLogSender = "Mkdir"
  51. symlinkLogSender = "Symlink"
  52. removeLogSender = "Remove"
  53. chownLogSender = "Chown"
  54. chmodLogSender = "Chmod"
  55. chtimesLogSender = "Chtimes"
  56. copyLogSender = "Copy"
  57. truncateLogSender = "Truncate"
  58. operationDownload = "download"
  59. operationUpload = "upload"
  60. operationFirstDownload = "first-download"
  61. operationFirstUpload = "first-upload"
  62. operationDelete = "delete"
  63. operationCopy = "copy"
  64. // Pre-download action name
  65. OperationPreDownload = "pre-download"
  66. // Pre-upload action name
  67. OperationPreUpload = "pre-upload"
  68. operationPreDelete = "pre-delete"
  69. operationRename = "rename"
  70. operationMkdir = "mkdir"
  71. operationRmdir = "rmdir"
  72. // SSH command action name
  73. OperationSSHCmd = "ssh_cmd"
  74. chtimesFormat = "2006-01-02T15:04:05" // YYYY-MM-DDTHH:MM:SS
  75. idleTimeoutCheckInterval = 3 * time.Minute
  76. periodicTimeoutCheckInterval = 1 * time.Minute
  77. )
  78. // Stat flags
  79. const (
  80. StatAttrUIDGID = 1
  81. StatAttrPerms = 2
  82. StatAttrTimes = 4
  83. StatAttrSize = 8
  84. )
  85. // Transfer types
  86. const (
  87. TransferUpload = iota
  88. TransferDownload
  89. )
  90. // Supported protocols
  91. const (
  92. ProtocolSFTP = "SFTP"
  93. ProtocolSCP = "SCP"
  94. ProtocolSSH = "SSH"
  95. ProtocolFTP = "FTP"
  96. ProtocolWebDAV = "DAV"
  97. ProtocolHTTP = "HTTP"
  98. ProtocolHTTPShare = "HTTPShare"
  99. ProtocolDataRetention = "DataRetention"
  100. ProtocolOIDC = "OIDC"
  101. protocolEventAction = "EventAction"
  102. )
  103. // Upload modes
  104. const (
  105. UploadModeStandard = 0
  106. UploadModeAtomic = 1
  107. UploadModeAtomicWithResume = 2
  108. UploadModeS3StoreOnError = 4
  109. UploadModeGCSStoreOnError = 8
  110. UploadModeAzureBlobStoreOnError = 16
  111. )
  112. func init() {
  113. Connections.clients = clientsMap{
  114. clients: make(map[string]int),
  115. }
  116. Connections.perUserConns = make(map[string]int)
  117. Connections.mapping = make(map[string]int)
  118. Connections.sshMapping = make(map[string]int)
  119. }
  120. // errors definitions
  121. var (
  122. ErrPermissionDenied = errors.New("permission denied")
  123. ErrNotExist = errors.New("no such file or directory")
  124. ErrOpUnsupported = errors.New("operation unsupported")
  125. ErrGenericFailure = errors.New("failure")
  126. ErrQuotaExceeded = errors.New("denying write due to space limit")
  127. ErrReadQuotaExceeded = errors.New("denying read due to quota limit")
  128. ErrConnectionDenied = errors.New("you are not allowed to connect")
  129. ErrNoBinding = errors.New("no binding configured")
  130. ErrCrtRevoked = errors.New("your certificate has been revoked")
  131. ErrNoCredentials = errors.New("no credential provided")
  132. ErrInternalFailure = errors.New("internal failure")
  133. ErrTransferAborted = errors.New("transfer aborted")
  134. ErrShuttingDown = errors.New("the service is shutting down")
  135. errNoTransfer = errors.New("requested transfer not found")
  136. errTransferMismatch = errors.New("transfer mismatch")
  137. )
  138. var (
  139. // Config is the configuration for the supported protocols
  140. Config Configuration
  141. // Connections is the list of active connections
  142. Connections ActiveConnections
  143. // QuotaScans is the list of active quota scans
  144. QuotaScans ActiveScans
  145. transfersChecker TransfersChecker
  146. supportedProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV,
  147. ProtocolHTTP, ProtocolHTTPShare, ProtocolOIDC}
  148. disconnHookProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP}
  149. // the map key is the protocol, for each protocol we can have multiple rate limiters
  150. rateLimiters map[string][]*rateLimiter
  151. isShuttingDown atomic.Bool
  152. ftpLoginCommands = []string{"PASS", "USER"}
  153. )
  154. // Initialize sets the common configuration
  155. func Initialize(c Configuration, isShared int) error {
  156. isShuttingDown.Store(false)
  157. util.SetUmask(c.Umask)
  158. Config = c
  159. Config.Actions.ExecuteOn = util.RemoveDuplicates(Config.Actions.ExecuteOn, true)
  160. Config.Actions.ExecuteSync = util.RemoveDuplicates(Config.Actions.ExecuteSync, true)
  161. Config.ProxyAllowed = util.RemoveDuplicates(Config.ProxyAllowed, true)
  162. Config.idleLoginTimeout = 2 * time.Minute
  163. Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute
  164. startPeriodicChecks(periodicTimeoutCheckInterval, isShared)
  165. Config.defender = nil
  166. Config.allowList = nil
  167. Config.rateLimitersList = nil
  168. rateLimiters = make(map[string][]*rateLimiter)
  169. for _, rlCfg := range c.RateLimitersConfig {
  170. if rlCfg.isEnabled() {
  171. if err := rlCfg.validate(); err != nil {
  172. return fmt.Errorf("rate limiters initialization error: %w", err)
  173. }
  174. rateLimiter := rlCfg.getLimiter()
  175. for _, protocol := range rlCfg.Protocols {
  176. rateLimiters[protocol] = append(rateLimiters[protocol], rateLimiter)
  177. }
  178. }
  179. }
  180. if len(rateLimiters) > 0 {
  181. rateLimitersList, err := dataprovider.NewIPList(dataprovider.IPListTypeRateLimiterSafeList)
  182. if err != nil {
  183. return fmt.Errorf("unable to initialize ratelimiters list: %w", err)
  184. }
  185. Config.rateLimitersList = rateLimitersList
  186. }
  187. if c.DefenderConfig.Enabled {
  188. if !util.Contains(supportedDefenderDrivers, c.DefenderConfig.Driver) {
  189. return fmt.Errorf("unsupported defender driver %q", c.DefenderConfig.Driver)
  190. }
  191. var defender Defender
  192. var err error
  193. switch c.DefenderConfig.Driver {
  194. case DefenderDriverProvider:
  195. defender, err = newDBDefender(&c.DefenderConfig)
  196. default:
  197. defender, err = newInMemoryDefender(&c.DefenderConfig)
  198. }
  199. if err != nil {
  200. return fmt.Errorf("defender initialization error: %v", err)
  201. }
  202. logger.Info(logSender, "", "defender initialized with config %+v", c.DefenderConfig)
  203. Config.defender = defender
  204. }
  205. if c.AllowListStatus > 0 {
  206. allowList, err := dataprovider.NewIPList(dataprovider.IPListTypeAllowList)
  207. if err != nil {
  208. return fmt.Errorf("unable to initialize the allow list: %w", err)
  209. }
  210. logger.Info(logSender, "", "allow list initialized")
  211. Config.allowList = allowList
  212. }
  213. if err := c.initializeProxyProtocol(); err != nil {
  214. return err
  215. }
  216. vfs.SetTempPath(c.TempPath)
  217. dataprovider.SetTempPath(c.TempPath)
  218. vfs.SetAllowSelfConnections(c.AllowSelfConnections)
  219. vfs.SetRenameMode(c.RenameMode)
  220. vfs.SetReadMetadataMode(c.Metadata.Read)
  221. vfs.SetResumeMaxSize(c.ResumeMaxSize)
  222. vfs.SetUploadMode(c.UploadMode)
  223. dataprovider.SetAllowSelfConnections(c.AllowSelfConnections)
  224. transfersChecker = getTransfersChecker(isShared)
  225. return nil
  226. }
  227. // CheckClosing returns an error if the service is closing
  228. func CheckClosing() error {
  229. if isShuttingDown.Load() {
  230. return ErrShuttingDown
  231. }
  232. return nil
  233. }
  234. // WaitForTransfers waits, for the specified grace time, for currently ongoing
  235. // client-initiated transfer sessions to completes.
  236. // A zero graceTime means no wait
  237. func WaitForTransfers(graceTime int) {
  238. if graceTime == 0 {
  239. return
  240. }
  241. if isShuttingDown.Swap(true) {
  242. return
  243. }
  244. if activeHooks.Load() == 0 && getActiveConnections() == 0 {
  245. return
  246. }
  247. graceTimer := time.NewTimer(time.Duration(graceTime) * time.Second)
  248. ticker := time.NewTicker(3 * time.Second)
  249. for {
  250. select {
  251. case <-ticker.C:
  252. hooks := activeHooks.Load()
  253. logger.Info(logSender, "", "active hooks: %d", hooks)
  254. if hooks == 0 && getActiveConnections() == 0 {
  255. logger.Info(logSender, "", "no more active connections, graceful shutdown")
  256. ticker.Stop()
  257. graceTimer.Stop()
  258. return
  259. }
  260. case <-graceTimer.C:
  261. logger.Info(logSender, "", "grace time expired, hard shutdown")
  262. ticker.Stop()
  263. return
  264. }
  265. }
  266. }
  267. // getActiveConnections returns the number of connections with active transfers
  268. func getActiveConnections() int {
  269. var activeConns int
  270. Connections.RLock()
  271. for _, c := range Connections.connections {
  272. if len(c.GetTransfers()) > 0 {
  273. activeConns++
  274. }
  275. }
  276. Connections.RUnlock()
  277. logger.Info(logSender, "", "number of connections with active transfers: %d", activeConns)
  278. return activeConns
  279. }
  280. // LimitRate blocks until all the configured rate limiters
  281. // allow one event to happen.
  282. // It returns an error if the time to wait exceeds the max
  283. // allowed delay
  284. func LimitRate(protocol, ip string) (time.Duration, error) {
  285. if Config.rateLimitersList != nil {
  286. isListed, _, err := Config.rateLimitersList.IsListed(ip, protocol)
  287. if err == nil && isListed {
  288. return 0, nil
  289. }
  290. }
  291. for _, limiter := range rateLimiters[protocol] {
  292. if delay, err := limiter.Wait(ip, protocol); err != nil {
  293. logger.Debug(logSender, "", "protocol %s ip %s: %v", protocol, ip, err)
  294. return delay, err
  295. }
  296. }
  297. return 0, nil
  298. }
  299. // Reload reloads the whitelist, the IP filter plugin and the defender's block and safe lists
  300. func Reload() error {
  301. plugin.Handler.ReloadFilter()
  302. return nil
  303. }
  304. // IsBanned returns true if the specified IP address is banned
  305. func IsBanned(ip, protocol string) bool {
  306. if plugin.Handler.IsIPBanned(ip, protocol) {
  307. return true
  308. }
  309. if Config.defender == nil {
  310. return false
  311. }
  312. return Config.defender.IsBanned(ip, protocol)
  313. }
  314. // GetDefenderBanTime returns the ban time for the given IP
  315. // or nil if the IP is not banned or the defender is disabled
  316. func GetDefenderBanTime(ip string) (*time.Time, error) {
  317. if Config.defender == nil {
  318. return nil, nil
  319. }
  320. return Config.defender.GetBanTime(ip)
  321. }
  322. // GetDefenderHosts returns hosts that are banned or for which some violations have been detected
  323. func GetDefenderHosts() ([]dataprovider.DefenderEntry, error) {
  324. if Config.defender == nil {
  325. return nil, nil
  326. }
  327. return Config.defender.GetHosts()
  328. }
  329. // GetDefenderHost returns a defender host by ip, if any
  330. func GetDefenderHost(ip string) (dataprovider.DefenderEntry, error) {
  331. if Config.defender == nil {
  332. return dataprovider.DefenderEntry{}, errors.New("defender is disabled")
  333. }
  334. return Config.defender.GetHost(ip)
  335. }
  336. // DeleteDefenderHost removes the specified IP address from the defender lists
  337. func DeleteDefenderHost(ip string) bool {
  338. if Config.defender == nil {
  339. return false
  340. }
  341. return Config.defender.DeleteHost(ip)
  342. }
  343. // GetDefenderScore returns the score for the given IP
  344. func GetDefenderScore(ip string) (int, error) {
  345. if Config.defender == nil {
  346. return 0, nil
  347. }
  348. return Config.defender.GetScore(ip)
  349. }
  350. // AddDefenderEvent adds the specified defender event for the given IP.
  351. // Returns true if the IP is in the defender's safe list.
  352. func AddDefenderEvent(ip, protocol string, event HostEvent) bool {
  353. if Config.defender == nil {
  354. return false
  355. }
  356. return Config.defender.AddEvent(ip, protocol, event)
  357. }
  358. func startPeriodicChecks(duration time.Duration, isShared int) {
  359. startEventScheduler()
  360. spec := fmt.Sprintf("@every %s", duration)
  361. _, err := eventScheduler.AddFunc(spec, Connections.checkTransfers)
  362. util.PanicOnError(err)
  363. logger.Info(logSender, "", "scheduled overquota transfers check, schedule %q", spec)
  364. if isShared == 1 {
  365. logger.Info(logSender, "", "add reload configs task")
  366. _, err := eventScheduler.AddFunc("@every 10m", smtp.ReloadProviderConf)
  367. util.PanicOnError(err)
  368. }
  369. if Config.IdleTimeout > 0 {
  370. ratio := idleTimeoutCheckInterval / periodicTimeoutCheckInterval
  371. spec = fmt.Sprintf("@every %s", duration*ratio)
  372. _, err = eventScheduler.AddFunc(spec, Connections.checkIdles)
  373. util.PanicOnError(err)
  374. logger.Info(logSender, "", "scheduled idle connections check, schedule %q", spec)
  375. }
  376. }
  377. // ActiveTransfer defines the interface for the current active transfers
  378. type ActiveTransfer interface {
  379. GetID() int64
  380. GetType() int
  381. GetSize() int64
  382. GetDownloadedSize() int64
  383. GetUploadedSize() int64
  384. GetVirtualPath() string
  385. GetStartTime() time.Time
  386. SignalClose(err error)
  387. Truncate(fsPath string, size int64) (int64, error)
  388. GetRealFsPath(fsPath string) string
  389. SetTimes(fsPath string, atime time.Time, mtime time.Time) bool
  390. GetTruncatedSize() int64
  391. HasSizeLimit() bool
  392. }
  393. // ActiveConnection defines the interface for the current active connections
  394. type ActiveConnection interface {
  395. GetID() string
  396. GetUsername() string
  397. GetRole() string
  398. GetMaxSessions() int
  399. GetLocalAddress() string
  400. GetRemoteAddress() string
  401. GetClientVersion() string
  402. GetProtocol() string
  403. GetConnectionTime() time.Time
  404. GetLastActivity() time.Time
  405. GetCommand() string
  406. Disconnect() error
  407. AddTransfer(t ActiveTransfer)
  408. RemoveTransfer(t ActiveTransfer)
  409. GetTransfers() []ConnectionTransfer
  410. SignalTransferClose(transferID int64, err error)
  411. CloseFS() error
  412. }
  413. // StatAttributes defines the attributes for set stat commands
  414. type StatAttributes struct {
  415. Mode os.FileMode
  416. Atime time.Time
  417. Mtime time.Time
  418. UID int
  419. GID int
  420. Flags int
  421. Size int64
  422. }
  423. // ConnectionTransfer defines the trasfer details
  424. type ConnectionTransfer struct {
  425. ID int64 `json:"-"`
  426. OperationType string `json:"operation_type"`
  427. StartTime int64 `json:"start_time"`
  428. Size int64 `json:"size"`
  429. VirtualPath string `json:"path"`
  430. HasSizeLimit bool `json:"-"`
  431. ULSize int64 `json:"-"`
  432. DLSize int64 `json:"-"`
  433. }
  434. // MetadataConfig defines how to handle metadata for cloud storage backends
  435. type MetadataConfig struct {
  436. // If not zero the metadata will be read before downloads and will be
  437. // available in notifications
  438. Read int `json:"read" mapstructure:"read"`
  439. }
  440. // Configuration defines configuration parameters common to all supported protocols
  441. type Configuration struct {
  442. // Maximum idle timeout as minutes. If a client is idle for a time that exceeds this setting it will be disconnected.
  443. // 0 means disabled
  444. IdleTimeout int `json:"idle_timeout" mapstructure:"idle_timeout"`
  445. // UploadMode 0 means standard, the files are uploaded directly to the requested path.
  446. // 1 means atomic: the files are uploaded to a temporary path and renamed to the requested path
  447. // when the client ends the upload. Atomic mode avoid problems such as a web server that
  448. // serves partial files when the files are being uploaded.
  449. // In atomic mode if there is an upload error the temporary file is deleted and so the requested
  450. // upload path will not contain a partial file.
  451. // 2 means atomic with resume support: as atomic but if there is an upload error the temporary
  452. // file is renamed to the requested path and not deleted, this way a client can reconnect and resume
  453. // the upload.
  454. // 4 means files for S3 backend are stored even if a client-side upload error is detected.
  455. // 8 means files for Google Cloud Storage backend are stored even if a client-side upload error is detected.
  456. // 16 means files for Azure Blob backend are stored even if a client-side upload error is detected.
  457. UploadMode int `json:"upload_mode" mapstructure:"upload_mode"`
  458. // Actions to execute for SFTP file operations and SSH commands
  459. Actions ProtocolActions `json:"actions" mapstructure:"actions"`
  460. // SetstatMode 0 means "normal mode": requests for changing permissions and owner/group are executed.
  461. // 1 means "ignore mode": requests for changing permissions and owner/group are silently ignored.
  462. // 2 means "ignore mode for cloud fs": requests for changing permissions and owner/group are
  463. // silently ignored for cloud based filesystem such as S3, GCS, Azure Blob. Requests for changing
  464. // modification times are ignored for cloud based filesystem if they are not supported.
  465. SetstatMode int `json:"setstat_mode" mapstructure:"setstat_mode"`
  466. // RenameMode defines how to handle directory renames. By default, renaming of non-empty directories
  467. // is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to 1 to enable recursive
  468. // renames for these providers, they may be slow, there is no atomic rename API like for local
  469. // filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry
  470. RenameMode int `json:"rename_mode" mapstructure:"rename_mode"`
  471. // ResumeMaxSize defines the maximum size allowed, in bytes, to resume uploads on storage backends
  472. // with immutable objects. By default, resuming uploads is not allowed for cloud storage providers
  473. // (S3, GCS, Azure Blob) because SFTPGo must rewrite the entire file.
  474. // Set to a value greater than 0 to allow resuming uploads of files smaller than or equal to the
  475. // defined size.
  476. ResumeMaxSize int64 `json:"resume_max_size" mapstructure:"resume_max_size"`
  477. // TempPath defines the path for temporary files such as those used for atomic uploads or file pipes.
  478. // If you set this option you must make sure that the defined path exists, is accessible for writing
  479. // by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise
  480. // the renaming for atomic uploads will become a copy and therefore may take a long time.
  481. // The temporary files are not namespaced. The default is generally fine. Leave empty for the default.
  482. TempPath string `json:"temp_path" mapstructure:"temp_path"`
  483. // Support for HAProxy PROXY protocol.
  484. // If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable
  485. // the proxy protocol. It provides a convenient way to safely transport connection information
  486. // such as a client's address across multiple layers of NAT or TCP proxies to get the real
  487. // client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported.
  488. // - 0 means disabled
  489. // - 1 means proxy protocol enabled. Proxy header will be used and requests without proxy header will be accepted.
  490. // - 2 means proxy protocol required. Proxy header will be used and requests without proxy header will be rejected.
  491. // If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too,
  492. // for example for HAProxy add "send-proxy" or "send-proxy-v2" to each server configuration line.
  493. ProxyProtocol int `json:"proxy_protocol" mapstructure:"proxy_protocol"`
  494. // List of IP addresses and IP ranges allowed to send the proxy header.
  495. // If proxy protocol is set to 1 and we receive a proxy header from an IP that is not in the list then the
  496. // connection will be accepted and the header will be ignored.
  497. // If proxy protocol is set to 2 and we receive a proxy header from an IP that is not in the list then the
  498. // connection will be rejected.
  499. ProxyAllowed []string `json:"proxy_allowed" mapstructure:"proxy_allowed"`
  500. // List of IP addresses and IP ranges for which not to read the proxy header
  501. ProxySkipped []string `json:"proxy_skipped" mapstructure:"proxy_skipped"`
  502. // Absolute path to an external program or an HTTP URL to invoke as soon as SFTPGo starts.
  503. // If you define an HTTP URL it will be invoked using a `GET` request.
  504. // Please note that SFTPGo services may not yet be available when this hook is run.
  505. // Leave empty do disable.
  506. StartupHook string `json:"startup_hook" mapstructure:"startup_hook"`
  507. // Absolute path to an external program or an HTTP URL to invoke after a user connects
  508. // and before he tries to login. It allows you to reject the connection based on the source
  509. // ip address. Leave empty do disable.
  510. PostConnectHook string `json:"post_connect_hook" mapstructure:"post_connect_hook"`
  511. // Absolute path to an external program or an HTTP URL to invoke after an SSH/FTP connection ends.
  512. // Leave empty do disable.
  513. PostDisconnectHook string `json:"post_disconnect_hook" mapstructure:"post_disconnect_hook"`
  514. // Absolute path to an external program or an HTTP URL to invoke after a data retention check completes.
  515. // Leave empty do disable.
  516. DataRetentionHook string `json:"data_retention_hook" mapstructure:"data_retention_hook"`
  517. // Maximum number of concurrent client connections. 0 means unlimited
  518. MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"`
  519. // Maximum number of concurrent client connections from the same host (IP). 0 means unlimited
  520. MaxPerHostConnections int `json:"max_per_host_connections" mapstructure:"max_per_host_connections"`
  521. // Defines the status of the global allow list. 0 means disabled, 1 enabled.
  522. // If enabled, only the listed IPs/networks can access the configured services, all other
  523. // client connections will be dropped before they even try to authenticate.
  524. // Ensure to enable this setting only after adding some allowed ip/networks from the WebAdmin/REST API
  525. AllowListStatus int `json:"allowlist_status" mapstructure:"allowlist_status"`
  526. // Allow users on this instance to use other users/virtual folders on this instance as storage backend.
  527. // Enable this setting if you know what you are doing.
  528. AllowSelfConnections int `json:"allow_self_connections" mapstructure:"allow_self_connections"`
  529. // Defender configuration
  530. DefenderConfig DefenderConfig `json:"defender" mapstructure:"defender"`
  531. // Rate limiter configurations
  532. RateLimitersConfig []RateLimiterConfig `json:"rate_limiters" mapstructure:"rate_limiters"`
  533. // Umask for new uploads. Leave blank to use the system default.
  534. Umask string `json:"umask" mapstructure:"umask"`
  535. // Metadata configuration
  536. Metadata MetadataConfig `json:"metadata" mapstructure:"metadata"`
  537. idleTimeoutAsDuration time.Duration
  538. idleLoginTimeout time.Duration
  539. defender Defender
  540. allowList *dataprovider.IPList
  541. rateLimitersList *dataprovider.IPList
  542. proxyAllowed []func(net.IP) bool
  543. proxySkipped []func(net.IP) bool
  544. }
  545. // IsAtomicUploadEnabled returns true if atomic upload is enabled
  546. func (c *Configuration) IsAtomicUploadEnabled() bool {
  547. return c.UploadMode&UploadModeAtomic != 0 || c.UploadMode&UploadModeAtomicWithResume != 0
  548. }
  549. func (c *Configuration) initializeProxyProtocol() error {
  550. if c.ProxyProtocol > 0 {
  551. allowed, err := util.ParseAllowedIPAndRanges(c.ProxyAllowed)
  552. if err != nil {
  553. return fmt.Errorf("invalid proxy allowed: %w", err)
  554. }
  555. skipped, err := util.ParseAllowedIPAndRanges(c.ProxySkipped)
  556. if err != nil {
  557. return fmt.Errorf("invalid proxy skipped: %w", err)
  558. }
  559. Config.proxyAllowed = allowed
  560. Config.proxySkipped = skipped
  561. }
  562. return nil
  563. }
  564. // GetProxyListener returns a wrapper for the given listener that supports the
  565. // HAProxy Proxy Protocol
  566. func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Listener, error) {
  567. if c.ProxyProtocol > 0 {
  568. defaultPolicy := proxyproto.REQUIRE
  569. if c.ProxyProtocol == 1 {
  570. defaultPolicy = proxyproto.IGNORE
  571. }
  572. return &proxyproto.Listener{
  573. Listener: listener,
  574. Policy: getProxyPolicy(c.proxyAllowed, c.proxySkipped, defaultPolicy),
  575. ReadHeaderTimeout: 10 * time.Second,
  576. }, nil
  577. }
  578. return nil, errors.New("proxy protocol not configured")
  579. }
  580. // GetRateLimitersStatus returns the rate limiters status
  581. func (c *Configuration) GetRateLimitersStatus() (bool, []string) {
  582. enabled := false
  583. var protocols []string
  584. for _, rlCfg := range c.RateLimitersConfig {
  585. if rlCfg.isEnabled() {
  586. enabled = true
  587. protocols = append(protocols, rlCfg.Protocols...)
  588. }
  589. }
  590. return enabled, util.RemoveDuplicates(protocols, false)
  591. }
  592. // IsAllowListEnabled returns true if the global allow list is enabled
  593. func (c *Configuration) IsAllowListEnabled() bool {
  594. return c.AllowListStatus > 0
  595. }
  596. // ExecuteStartupHook runs the startup hook if defined
  597. func (c *Configuration) ExecuteStartupHook() error {
  598. if c.StartupHook == "" {
  599. return nil
  600. }
  601. if strings.HasPrefix(c.StartupHook, "http") {
  602. var url *url.URL
  603. url, err := url.Parse(c.StartupHook)
  604. if err != nil {
  605. logger.Warn(logSender, "", "Invalid startup hook %q: %v", c.StartupHook, err)
  606. return err
  607. }
  608. startTime := time.Now()
  609. resp, err := httpclient.RetryableGet(url.String())
  610. if err != nil {
  611. logger.Warn(logSender, "", "Error executing startup hook: %v", err)
  612. return err
  613. }
  614. defer resp.Body.Close()
  615. logger.Debug(logSender, "", "Startup hook executed, elapsed: %v, response code: %v", time.Since(startTime), resp.StatusCode)
  616. return nil
  617. }
  618. if !filepath.IsAbs(c.StartupHook) {
  619. err := fmt.Errorf("invalid startup hook %q", c.StartupHook)
  620. logger.Warn(logSender, "", "Invalid startup hook %q", c.StartupHook)
  621. return err
  622. }
  623. startTime := time.Now()
  624. timeout, env, args := command.GetConfig(c.StartupHook, command.HookStartup)
  625. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  626. defer cancel()
  627. cmd := exec.CommandContext(ctx, c.StartupHook, args...)
  628. cmd.Env = env
  629. err := cmd.Run()
  630. logger.Debug(logSender, "", "Startup hook executed, elapsed: %s, error: %v", time.Since(startTime), err)
  631. return nil
  632. }
  633. func (c *Configuration) executePostDisconnectHook(remoteAddr, protocol, username, connID string, connectionTime time.Time) {
  634. startNewHook()
  635. defer hookEnded()
  636. ipAddr := util.GetIPFromRemoteAddress(remoteAddr)
  637. connDuration := int64(time.Since(connectionTime) / time.Millisecond)
  638. if strings.HasPrefix(c.PostDisconnectHook, "http") {
  639. var url *url.URL
  640. url, err := url.Parse(c.PostDisconnectHook)
  641. if err != nil {
  642. logger.Warn(protocol, connID, "Invalid post disconnect hook %q: %v", c.PostDisconnectHook, err)
  643. return
  644. }
  645. q := url.Query()
  646. q.Add("ip", ipAddr)
  647. q.Add("protocol", protocol)
  648. q.Add("username", username)
  649. q.Add("connection_duration", strconv.FormatInt(connDuration, 10))
  650. url.RawQuery = q.Encode()
  651. startTime := time.Now()
  652. resp, err := httpclient.RetryableGet(url.String())
  653. respCode := 0
  654. if err == nil {
  655. respCode = resp.StatusCode
  656. resp.Body.Close()
  657. }
  658. logger.Debug(protocol, connID, "Post disconnect hook response code: %v, elapsed: %v, err: %v",
  659. respCode, time.Since(startTime), err)
  660. return
  661. }
  662. if !filepath.IsAbs(c.PostDisconnectHook) {
  663. logger.Debug(protocol, connID, "invalid post disconnect hook %q", c.PostDisconnectHook)
  664. return
  665. }
  666. timeout, env, args := command.GetConfig(c.PostDisconnectHook, command.HookPostDisconnect)
  667. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  668. defer cancel()
  669. startTime := time.Now()
  670. cmd := exec.CommandContext(ctx, c.PostDisconnectHook, args...)
  671. cmd.Env = append(env,
  672. fmt.Sprintf("SFTPGO_CONNECTION_IP=%s", ipAddr),
  673. fmt.Sprintf("SFTPGO_CONNECTION_USERNAME=%s", username),
  674. fmt.Sprintf("SFTPGO_CONNECTION_DURATION=%d", connDuration),
  675. fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%s", protocol))
  676. err := cmd.Run()
  677. logger.Debug(protocol, connID, "Post disconnect hook executed, elapsed: %s error: %v", time.Since(startTime), err)
  678. }
  679. func (c *Configuration) checkPostDisconnectHook(remoteAddr, protocol, username, connID string, connectionTime time.Time) {
  680. if c.PostDisconnectHook == "" {
  681. return
  682. }
  683. if !util.Contains(disconnHookProtocols, protocol) {
  684. return
  685. }
  686. go c.executePostDisconnectHook(remoteAddr, protocol, username, connID, connectionTime)
  687. }
  688. // ExecutePostConnectHook executes the post connect hook if defined
  689. func (c *Configuration) ExecutePostConnectHook(ipAddr, protocol string) error {
  690. if c.PostConnectHook == "" {
  691. return nil
  692. }
  693. if strings.HasPrefix(c.PostConnectHook, "http") {
  694. var url *url.URL
  695. url, err := url.Parse(c.PostConnectHook)
  696. if err != nil {
  697. logger.Warn(protocol, "", "Login from ip %q denied, invalid post connect hook %q: %v",
  698. ipAddr, c.PostConnectHook, err)
  699. return getPermissionDeniedError(protocol)
  700. }
  701. q := url.Query()
  702. q.Add("ip", ipAddr)
  703. q.Add("protocol", protocol)
  704. url.RawQuery = q.Encode()
  705. resp, err := httpclient.RetryableGet(url.String())
  706. if err != nil {
  707. logger.Warn(protocol, "", "Login from ip %q denied, error executing post connect hook: %v", ipAddr, err)
  708. return getPermissionDeniedError(protocol)
  709. }
  710. defer resp.Body.Close()
  711. if resp.StatusCode != http.StatusOK {
  712. logger.Warn(protocol, "", "Login from ip %q denied, post connect hook response code: %v", ipAddr, resp.StatusCode)
  713. return getPermissionDeniedError(protocol)
  714. }
  715. return nil
  716. }
  717. if !filepath.IsAbs(c.PostConnectHook) {
  718. err := fmt.Errorf("invalid post connect hook %q", c.PostConnectHook)
  719. logger.Warn(protocol, "", "Login from ip %q denied: %v", ipAddr, err)
  720. return getPermissionDeniedError(protocol)
  721. }
  722. timeout, env, args := command.GetConfig(c.PostConnectHook, command.HookPostConnect)
  723. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  724. defer cancel()
  725. cmd := exec.CommandContext(ctx, c.PostConnectHook, args...)
  726. cmd.Env = append(env,
  727. fmt.Sprintf("SFTPGO_CONNECTION_IP=%s", ipAddr),
  728. fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%s", protocol))
  729. err := cmd.Run()
  730. if err != nil {
  731. logger.Warn(protocol, "", "Login from ip %q denied, connect hook error: %v", ipAddr, err)
  732. return getPermissionDeniedError(protocol)
  733. }
  734. return nil
  735. }
  736. func getProxyPolicy(allowed, skipped []func(net.IP) bool, def proxyproto.Policy) proxyproto.PolicyFunc {
  737. return func(upstream net.Addr) (proxyproto.Policy, error) {
  738. upstreamIP, err := util.GetIPFromNetAddr(upstream)
  739. if err != nil {
  740. // something is wrong with the source IP, better reject the connection
  741. return proxyproto.REJECT, err
  742. }
  743. for _, skippedFrom := range skipped {
  744. if skippedFrom(upstreamIP) {
  745. return proxyproto.SKIP, nil
  746. }
  747. }
  748. for _, allowFrom := range allowed {
  749. if allowFrom(upstreamIP) {
  750. if def == proxyproto.REQUIRE {
  751. return proxyproto.REQUIRE, nil
  752. }
  753. return proxyproto.USE, nil
  754. }
  755. }
  756. return def, nil
  757. }
  758. }
  759. // SSHConnection defines an ssh connection.
  760. // Each SSH connection can open several channels for SFTP or SSH commands
  761. type SSHConnection struct {
  762. id string
  763. conn net.Conn
  764. lastActivity atomic.Int64
  765. }
  766. // NewSSHConnection returns a new SSHConnection
  767. func NewSSHConnection(id string, conn net.Conn) *SSHConnection {
  768. c := &SSHConnection{
  769. id: id,
  770. conn: conn,
  771. }
  772. c.lastActivity.Store(time.Now().UnixNano())
  773. return c
  774. }
  775. // GetID returns the ID for this SSHConnection
  776. func (c *SSHConnection) GetID() string {
  777. return c.id
  778. }
  779. // UpdateLastActivity updates last activity for this connection
  780. func (c *SSHConnection) UpdateLastActivity() {
  781. c.lastActivity.Store(time.Now().UnixNano())
  782. }
  783. // GetLastActivity returns the last connection activity
  784. func (c *SSHConnection) GetLastActivity() time.Time {
  785. return time.Unix(0, c.lastActivity.Load())
  786. }
  787. // Close closes the underlying network connection
  788. func (c *SSHConnection) Close() error {
  789. return c.conn.Close()
  790. }
  791. // ActiveConnections holds the currect active connections with the associated transfers
  792. type ActiveConnections struct {
  793. // clients contains both authenticated and estabilished connections and the ones waiting
  794. // for authentication
  795. clients clientsMap
  796. transfersCheckStatus atomic.Bool
  797. sync.RWMutex
  798. connections []ActiveConnection
  799. mapping map[string]int
  800. sshConnections []*SSHConnection
  801. sshMapping map[string]int
  802. perUserConns map[string]int
  803. }
  804. // internal method, must be called within a locked block
  805. func (conns *ActiveConnections) addUserConnection(username string) {
  806. if username == "" {
  807. return
  808. }
  809. conns.perUserConns[username]++
  810. }
  811. // internal method, must be called within a locked block
  812. func (conns *ActiveConnections) removeUserConnection(username string) {
  813. if username == "" {
  814. return
  815. }
  816. if val, ok := conns.perUserConns[username]; ok {
  817. conns.perUserConns[username]--
  818. if val > 1 {
  819. return
  820. }
  821. delete(conns.perUserConns, username)
  822. }
  823. }
  824. // GetActiveSessions returns the number of active sessions for the given username.
  825. // We return the open sessions for any protocol
  826. func (conns *ActiveConnections) GetActiveSessions(username string) int {
  827. conns.RLock()
  828. defer conns.RUnlock()
  829. return conns.perUserConns[username]
  830. }
  831. // Add adds a new connection to the active ones
  832. func (conns *ActiveConnections) Add(c ActiveConnection) error {
  833. conns.Lock()
  834. defer conns.Unlock()
  835. if username := c.GetUsername(); username != "" {
  836. if maxSessions := c.GetMaxSessions(); maxSessions > 0 {
  837. if val := conns.perUserConns[username]; val >= maxSessions {
  838. return fmt.Errorf("too many open sessions: %d/%d", val, maxSessions)
  839. }
  840. }
  841. conns.addUserConnection(username)
  842. }
  843. conns.mapping[c.GetID()] = len(conns.connections)
  844. conns.connections = append(conns.connections, c)
  845. metric.UpdateActiveConnectionsSize(len(conns.connections))
  846. logger.Debug(c.GetProtocol(), c.GetID(), "connection added, local address %q, remote address %q, num open connections: %d",
  847. c.GetLocalAddress(), c.GetRemoteAddress(), len(conns.connections))
  848. return nil
  849. }
  850. // Swap replaces an existing connection with the given one.
  851. // This method is useful if you have to change some connection details
  852. // for example for FTP is used to update the connection once the user
  853. // authenticates
  854. func (conns *ActiveConnections) Swap(c ActiveConnection) error {
  855. conns.Lock()
  856. defer conns.Unlock()
  857. if idx, ok := conns.mapping[c.GetID()]; ok {
  858. conn := conns.connections[idx]
  859. conns.removeUserConnection(conn.GetUsername())
  860. if username := c.GetUsername(); username != "" {
  861. if maxSessions := c.GetMaxSessions(); maxSessions > 0 {
  862. if val, ok := conns.perUserConns[username]; ok && val >= maxSessions {
  863. conns.addUserConnection(conn.GetUsername())
  864. return fmt.Errorf("too many open sessions: %d/%d", val, maxSessions)
  865. }
  866. }
  867. conns.addUserConnection(username)
  868. }
  869. err := conn.CloseFS()
  870. conns.connections[idx] = c
  871. logger.Debug(logSender, c.GetID(), "connection swapped, close fs error: %v", err)
  872. conn = nil
  873. return nil
  874. }
  875. return errors.New("connection to swap not found")
  876. }
  877. // Remove removes a connection from the active ones
  878. func (conns *ActiveConnections) Remove(connectionID string) {
  879. conns.Lock()
  880. defer conns.Unlock()
  881. if idx, ok := conns.mapping[connectionID]; ok {
  882. conn := conns.connections[idx]
  883. err := conn.CloseFS()
  884. lastIdx := len(conns.connections) - 1
  885. conns.connections[idx] = conns.connections[lastIdx]
  886. conns.connections[lastIdx] = nil
  887. conns.connections = conns.connections[:lastIdx]
  888. delete(conns.mapping, connectionID)
  889. if idx != lastIdx {
  890. conns.mapping[conns.connections[idx].GetID()] = idx
  891. }
  892. conns.removeUserConnection(conn.GetUsername())
  893. metric.UpdateActiveConnectionsSize(lastIdx)
  894. logger.Debug(conn.GetProtocol(), conn.GetID(), "connection removed, local address %q, remote address %q close fs error: %v, num open connections: %d",
  895. conn.GetLocalAddress(), conn.GetRemoteAddress(), err, lastIdx)
  896. if conn.GetProtocol() == ProtocolFTP && conn.GetUsername() == "" && !util.Contains(ftpLoginCommands, conn.GetCommand()) {
  897. ip := util.GetIPFromRemoteAddress(conn.GetRemoteAddress())
  898. logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTried, ProtocolFTP,
  899. dataprovider.ErrNoAuthTried.Error())
  900. metric.AddNoAuthTried()
  901. AddDefenderEvent(ip, ProtocolFTP, HostEventNoLoginTried)
  902. dataprovider.ExecutePostLoginHook(&dataprovider.User{}, dataprovider.LoginMethodNoAuthTried, ip,
  903. ProtocolFTP, dataprovider.ErrNoAuthTried)
  904. plugin.Handler.NotifyLogEvent(notifier.LogEventTypeNoLoginTried, ProtocolFTP, "", ip, "",
  905. dataprovider.ErrNoAuthTried)
  906. }
  907. Config.checkPostDisconnectHook(conn.GetRemoteAddress(), conn.GetProtocol(), conn.GetUsername(),
  908. conn.GetID(), conn.GetConnectionTime())
  909. return
  910. }
  911. logger.Debug(logSender, "", "connection id %q to remove not found!", connectionID)
  912. }
  913. // Close closes an active connection.
  914. // It returns true on success
  915. func (conns *ActiveConnections) Close(connectionID, role string) bool {
  916. conns.RLock()
  917. var result bool
  918. if idx, ok := conns.mapping[connectionID]; ok {
  919. c := conns.connections[idx]
  920. if role == "" || c.GetRole() == role {
  921. defer func(conn ActiveConnection) {
  922. err := conn.Disconnect()
  923. logger.Debug(conn.GetProtocol(), conn.GetID(), "close connection requested, close err: %v", err)
  924. }(c)
  925. result = true
  926. }
  927. }
  928. conns.RUnlock()
  929. return result
  930. }
  931. // AddSSHConnection adds a new ssh connection to the active ones
  932. func (conns *ActiveConnections) AddSSHConnection(c *SSHConnection) {
  933. conns.Lock()
  934. defer conns.Unlock()
  935. conns.sshMapping[c.GetID()] = len(conns.sshConnections)
  936. conns.sshConnections = append(conns.sshConnections, c)
  937. logger.Debug(logSender, c.GetID(), "ssh connection added, num open connections: %d", len(conns.sshConnections))
  938. }
  939. // RemoveSSHConnection removes a connection from the active ones
  940. func (conns *ActiveConnections) RemoveSSHConnection(connectionID string) {
  941. conns.Lock()
  942. defer conns.Unlock()
  943. if idx, ok := conns.sshMapping[connectionID]; ok {
  944. lastIdx := len(conns.sshConnections) - 1
  945. conns.sshConnections[idx] = conns.sshConnections[lastIdx]
  946. conns.sshConnections[lastIdx] = nil
  947. conns.sshConnections = conns.sshConnections[:lastIdx]
  948. delete(conns.sshMapping, connectionID)
  949. if idx != lastIdx {
  950. conns.sshMapping[conns.sshConnections[idx].GetID()] = idx
  951. }
  952. logger.Debug(logSender, connectionID, "ssh connection removed, num open ssh connections: %d", lastIdx)
  953. return
  954. }
  955. logger.Warn(logSender, "", "ssh connection to remove with id %q not found!", connectionID)
  956. }
  957. func (conns *ActiveConnections) checkIdles() {
  958. conns.RLock()
  959. for _, sshConn := range conns.sshConnections {
  960. idleTime := time.Since(sshConn.GetLastActivity())
  961. if idleTime > Config.idleTimeoutAsDuration {
  962. // we close an SSH connection if it has no active connections associated
  963. idToMatch := fmt.Sprintf("_%s_", sshConn.GetID())
  964. toClose := true
  965. for _, conn := range conns.connections {
  966. if strings.Contains(conn.GetID(), idToMatch) {
  967. if time.Since(conn.GetLastActivity()) <= Config.idleTimeoutAsDuration {
  968. toClose = false
  969. break
  970. }
  971. }
  972. }
  973. if toClose {
  974. defer func(c *SSHConnection) {
  975. err := c.Close()
  976. logger.Debug(logSender, c.GetID(), "close idle SSH connection, idle time: %v, close err: %v",
  977. time.Since(c.GetLastActivity()), err)
  978. }(sshConn)
  979. }
  980. }
  981. }
  982. for _, c := range conns.connections {
  983. idleTime := time.Since(c.GetLastActivity())
  984. isUnauthenticatedFTPUser := (c.GetProtocol() == ProtocolFTP && c.GetUsername() == "")
  985. if idleTime > Config.idleTimeoutAsDuration || (isUnauthenticatedFTPUser && idleTime > Config.idleLoginTimeout) {
  986. defer func(conn ActiveConnection) {
  987. err := conn.Disconnect()
  988. logger.Debug(conn.GetProtocol(), conn.GetID(), "close idle connection, idle time: %v, username: %q close err: %v",
  989. time.Since(conn.GetLastActivity()), conn.GetUsername(), err)
  990. }(c)
  991. }
  992. }
  993. conns.RUnlock()
  994. }
  995. func (conns *ActiveConnections) checkTransfers() {
  996. if conns.transfersCheckStatus.Load() {
  997. logger.Warn(logSender, "", "the previous transfer check is still running, skipping execution")
  998. return
  999. }
  1000. conns.transfersCheckStatus.Store(true)
  1001. defer conns.transfersCheckStatus.Store(false)
  1002. conns.RLock()
  1003. if len(conns.connections) < 2 {
  1004. conns.RUnlock()
  1005. return
  1006. }
  1007. var wg sync.WaitGroup
  1008. logger.Debug(logSender, "", "start concurrent transfers check")
  1009. // update the current size for transfers to monitors
  1010. for _, c := range conns.connections {
  1011. for _, t := range c.GetTransfers() {
  1012. if t.HasSizeLimit {
  1013. wg.Add(1)
  1014. go func(transfer ConnectionTransfer, connID string) {
  1015. defer wg.Done()
  1016. transfersChecker.UpdateTransferCurrentSizes(transfer.ULSize, transfer.DLSize, transfer.ID, connID)
  1017. }(t, c.GetID())
  1018. }
  1019. }
  1020. }
  1021. conns.RUnlock()
  1022. logger.Debug(logSender, "", "waiting for the update of the transfers current size")
  1023. wg.Wait()
  1024. logger.Debug(logSender, "", "getting overquota transfers")
  1025. overquotaTransfers := transfersChecker.GetOverquotaTransfers()
  1026. logger.Debug(logSender, "", "number of overquota transfers: %v", len(overquotaTransfers))
  1027. if len(overquotaTransfers) == 0 {
  1028. return
  1029. }
  1030. conns.RLock()
  1031. defer conns.RUnlock()
  1032. for _, c := range conns.connections {
  1033. for _, overquotaTransfer := range overquotaTransfers {
  1034. if c.GetID() == overquotaTransfer.ConnID {
  1035. logger.Info(logSender, c.GetID(), "user %q is overquota, try to close transfer id %v",
  1036. c.GetUsername(), overquotaTransfer.TransferID)
  1037. var err error
  1038. if overquotaTransfer.TransferType == TransferDownload {
  1039. err = getReadQuotaExceededError(c.GetProtocol())
  1040. } else {
  1041. err = getQuotaExceededError(c.GetProtocol())
  1042. }
  1043. c.SignalTransferClose(overquotaTransfer.TransferID, err)
  1044. }
  1045. }
  1046. }
  1047. logger.Debug(logSender, "", "transfers check completed")
  1048. }
  1049. // AddClientConnection stores a new client connection
  1050. func (conns *ActiveConnections) AddClientConnection(ipAddr string) {
  1051. conns.clients.add(ipAddr)
  1052. }
  1053. // RemoveClientConnection removes a disconnected client from the tracked ones
  1054. func (conns *ActiveConnections) RemoveClientConnection(ipAddr string) {
  1055. conns.clients.remove(ipAddr)
  1056. }
  1057. // GetClientConnections returns the total number of client connections
  1058. func (conns *ActiveConnections) GetClientConnections() int32 {
  1059. return conns.clients.getTotal()
  1060. }
  1061. // IsNewConnectionAllowed returns an error if the maximum number of concurrent allowed
  1062. // connections is exceeded or a whitelist is defined and the specified ipAddr is not listed
  1063. // or the service is shutting down
  1064. func (conns *ActiveConnections) IsNewConnectionAllowed(ipAddr, protocol string) error {
  1065. if isShuttingDown.Load() {
  1066. return ErrShuttingDown
  1067. }
  1068. if Config.allowList != nil {
  1069. isListed, _, err := Config.allowList.IsListed(ipAddr, protocol)
  1070. if err != nil {
  1071. logger.Error(logSender, "", "unable to query allow list, connection denied, ip %q, protocol %s, err: %v",
  1072. ipAddr, protocol, err)
  1073. return ErrConnectionDenied
  1074. }
  1075. if !isListed {
  1076. return ErrConnectionDenied
  1077. }
  1078. }
  1079. if Config.MaxTotalConnections == 0 && Config.MaxPerHostConnections == 0 {
  1080. return nil
  1081. }
  1082. if Config.MaxPerHostConnections > 0 {
  1083. if total := conns.clients.getTotalFrom(ipAddr); total > Config.MaxPerHostConnections {
  1084. if !AddDefenderEvent(ipAddr, protocol, HostEventLimitExceeded) {
  1085. logger.Warn(logSender, "", "connection denied, active connections from IP %q: %d/%d",
  1086. ipAddr, total, Config.MaxPerHostConnections)
  1087. return ErrConnectionDenied
  1088. }
  1089. logger.Info(logSender, "", "active connections from safe IP %q: %d", ipAddr, total)
  1090. }
  1091. }
  1092. if Config.MaxTotalConnections > 0 {
  1093. if total := conns.clients.getTotal(); total > int32(Config.MaxTotalConnections) {
  1094. logger.Info(logSender, "", "active client connections %d/%d", total, Config.MaxTotalConnections)
  1095. return ErrConnectionDenied
  1096. }
  1097. // on a single SFTP connection we could have multiple SFTP channels or commands
  1098. // so we check the estabilished connections too
  1099. conns.RLock()
  1100. defer conns.RUnlock()
  1101. if sess := len(conns.connections); sess >= Config.MaxTotalConnections {
  1102. logger.Info(logSender, "", "active client sessions %d/%d", sess, Config.MaxTotalConnections)
  1103. return ErrConnectionDenied
  1104. }
  1105. }
  1106. return nil
  1107. }
  1108. // GetStats returns stats for active connections
  1109. func (conns *ActiveConnections) GetStats(role string) []ConnectionStatus {
  1110. conns.RLock()
  1111. defer conns.RUnlock()
  1112. stats := make([]ConnectionStatus, 0, len(conns.connections))
  1113. node := dataprovider.GetNodeName()
  1114. for _, c := range conns.connections {
  1115. if role == "" || c.GetRole() == role {
  1116. stat := ConnectionStatus{
  1117. Username: c.GetUsername(),
  1118. ConnectionID: c.GetID(),
  1119. ClientVersion: c.GetClientVersion(),
  1120. RemoteAddress: c.GetRemoteAddress(),
  1121. ConnectionTime: util.GetTimeAsMsSinceEpoch(c.GetConnectionTime()),
  1122. LastActivity: util.GetTimeAsMsSinceEpoch(c.GetLastActivity()),
  1123. CurrentTime: util.GetTimeAsMsSinceEpoch(time.Now()),
  1124. Protocol: c.GetProtocol(),
  1125. Command: c.GetCommand(),
  1126. Transfers: c.GetTransfers(),
  1127. Node: node,
  1128. }
  1129. stats = append(stats, stat)
  1130. }
  1131. }
  1132. return stats
  1133. }
  1134. // ConnectionStatus returns the status for an active connection
  1135. type ConnectionStatus struct {
  1136. // Logged in username
  1137. Username string `json:"username"`
  1138. // Unique identifier for the connection
  1139. ConnectionID string `json:"connection_id"`
  1140. // client's version string
  1141. ClientVersion string `json:"client_version,omitempty"`
  1142. // Remote address for this connection
  1143. RemoteAddress string `json:"remote_address"`
  1144. // Connection time as unix timestamp in milliseconds
  1145. ConnectionTime int64 `json:"connection_time"`
  1146. // Last activity as unix timestamp in milliseconds
  1147. LastActivity int64 `json:"last_activity"`
  1148. // Current time as unix timestamp in milliseconds
  1149. CurrentTime int64 `json:"current_time"`
  1150. // Protocol for this connection
  1151. Protocol string `json:"protocol"`
  1152. // active uploads/downloads
  1153. Transfers []ConnectionTransfer `json:"active_transfers,omitempty"`
  1154. // SSH command or WebDAV method
  1155. Command string `json:"command,omitempty"`
  1156. // Node identifier, omitted for single node installations
  1157. Node string `json:"node,omitempty"`
  1158. }
  1159. // ActiveQuotaScan defines an active quota scan for a user
  1160. type ActiveQuotaScan struct {
  1161. // Username to which the quota scan refers
  1162. Username string `json:"username"`
  1163. // quota scan start time as unix timestamp in milliseconds
  1164. StartTime int64 `json:"start_time"`
  1165. Role string `json:"-"`
  1166. }
  1167. // ActiveVirtualFolderQuotaScan defines an active quota scan for a virtual folder
  1168. type ActiveVirtualFolderQuotaScan struct {
  1169. // folder name to which the quota scan refers
  1170. Name string `json:"name"`
  1171. // quota scan start time as unix timestamp in milliseconds
  1172. StartTime int64 `json:"start_time"`
  1173. }
  1174. // ActiveScans holds the active quota scans
  1175. type ActiveScans struct {
  1176. sync.RWMutex
  1177. UserScans []ActiveQuotaScan
  1178. FolderScans []ActiveVirtualFolderQuotaScan
  1179. }
  1180. // GetUsersQuotaScans returns the active users quota scans
  1181. func (s *ActiveScans) GetUsersQuotaScans(role string) []ActiveQuotaScan {
  1182. s.RLock()
  1183. defer s.RUnlock()
  1184. scans := make([]ActiveQuotaScan, 0, len(s.UserScans))
  1185. for _, scan := range s.UserScans {
  1186. if role == "" || role == scan.Role {
  1187. scans = append(scans, ActiveQuotaScan{
  1188. Username: scan.Username,
  1189. StartTime: scan.StartTime,
  1190. })
  1191. }
  1192. }
  1193. return scans
  1194. }
  1195. // AddUserQuotaScan adds a user to the ones with active quota scans.
  1196. // Returns false if the user has a quota scan already running
  1197. func (s *ActiveScans) AddUserQuotaScan(username, role string) bool {
  1198. s.Lock()
  1199. defer s.Unlock()
  1200. for _, scan := range s.UserScans {
  1201. if scan.Username == username {
  1202. return false
  1203. }
  1204. }
  1205. s.UserScans = append(s.UserScans, ActiveQuotaScan{
  1206. Username: username,
  1207. StartTime: util.GetTimeAsMsSinceEpoch(time.Now()),
  1208. Role: role,
  1209. })
  1210. return true
  1211. }
  1212. // RemoveUserQuotaScan removes a user from the ones with active quota scans.
  1213. // Returns false if the user has no active quota scans
  1214. func (s *ActiveScans) RemoveUserQuotaScan(username string) bool {
  1215. s.Lock()
  1216. defer s.Unlock()
  1217. for idx, scan := range s.UserScans {
  1218. if scan.Username == username {
  1219. lastIdx := len(s.UserScans) - 1
  1220. s.UserScans[idx] = s.UserScans[lastIdx]
  1221. s.UserScans = s.UserScans[:lastIdx]
  1222. return true
  1223. }
  1224. }
  1225. return false
  1226. }
  1227. // GetVFoldersQuotaScans returns the active quota scans for virtual folders
  1228. func (s *ActiveScans) GetVFoldersQuotaScans() []ActiveVirtualFolderQuotaScan {
  1229. s.RLock()
  1230. defer s.RUnlock()
  1231. scans := make([]ActiveVirtualFolderQuotaScan, len(s.FolderScans))
  1232. copy(scans, s.FolderScans)
  1233. return scans
  1234. }
  1235. // AddVFolderQuotaScan adds a virtual folder to the ones with active quota scans.
  1236. // Returns false if the folder has a quota scan already running
  1237. func (s *ActiveScans) AddVFolderQuotaScan(folderName string) bool {
  1238. s.Lock()
  1239. defer s.Unlock()
  1240. for _, scan := range s.FolderScans {
  1241. if scan.Name == folderName {
  1242. return false
  1243. }
  1244. }
  1245. s.FolderScans = append(s.FolderScans, ActiveVirtualFolderQuotaScan{
  1246. Name: folderName,
  1247. StartTime: util.GetTimeAsMsSinceEpoch(time.Now()),
  1248. })
  1249. return true
  1250. }
  1251. // RemoveVFolderQuotaScan removes a folder from the ones with active quota scans.
  1252. // Returns false if the folder has no active quota scans
  1253. func (s *ActiveScans) RemoveVFolderQuotaScan(folderName string) bool {
  1254. s.Lock()
  1255. defer s.Unlock()
  1256. for idx, scan := range s.FolderScans {
  1257. if scan.Name == folderName {
  1258. lastIdx := len(s.FolderScans) - 1
  1259. s.FolderScans[idx] = s.FolderScans[lastIdx]
  1260. s.FolderScans = s.FolderScans[:lastIdx]
  1261. return true
  1262. }
  1263. }
  1264. return false
  1265. }