common.go 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412
  1. // Copyright (C) 2019 Nicola Murino
  2. //
  3. // This program is free software: you can redistribute it and/or modify
  4. // it under the terms of the GNU Affero General Public License as published
  5. // by the Free Software Foundation, version 3.
  6. //
  7. // This program is distributed in the hope that it will be useful,
  8. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. // GNU Affero General Public License for more details.
  11. //
  12. // You should have received a copy of the GNU Affero General Public License
  13. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  14. // Package common defines code shared among file transfer packages and protocols
  15. package common
  16. import (
  17. "context"
  18. "errors"
  19. "fmt"
  20. "net"
  21. "net/http"
  22. "net/url"
  23. "os"
  24. "os/exec"
  25. "path/filepath"
  26. "strconv"
  27. "strings"
  28. "sync"
  29. "sync/atomic"
  30. "time"
  31. "github.com/pires/go-proxyproto"
  32. "github.com/sftpgo/sdk/plugin/notifier"
  33. "github.com/drakkan/sftpgo/v2/internal/command"
  34. "github.com/drakkan/sftpgo/v2/internal/dataprovider"
  35. "github.com/drakkan/sftpgo/v2/internal/httpclient"
  36. "github.com/drakkan/sftpgo/v2/internal/logger"
  37. "github.com/drakkan/sftpgo/v2/internal/metric"
  38. "github.com/drakkan/sftpgo/v2/internal/plugin"
  39. "github.com/drakkan/sftpgo/v2/internal/smtp"
  40. "github.com/drakkan/sftpgo/v2/internal/util"
  41. "github.com/drakkan/sftpgo/v2/internal/version"
  42. "github.com/drakkan/sftpgo/v2/internal/vfs"
  43. )
  44. // constants
  45. const (
  46. logSender = "common"
  47. uploadLogSender = "Upload"
  48. downloadLogSender = "Download"
  49. renameLogSender = "Rename"
  50. rmdirLogSender = "Rmdir"
  51. mkdirLogSender = "Mkdir"
  52. symlinkLogSender = "Symlink"
  53. removeLogSender = "Remove"
  54. chownLogSender = "Chown"
  55. chmodLogSender = "Chmod"
  56. chtimesLogSender = "Chtimes"
  57. copyLogSender = "Copy"
  58. truncateLogSender = "Truncate"
  59. operationDownload = "download"
  60. operationUpload = "upload"
  61. operationFirstDownload = "first-download"
  62. operationFirstUpload = "first-upload"
  63. operationDelete = "delete"
  64. operationCopy = "copy"
  65. // Pre-download action name
  66. OperationPreDownload = "pre-download"
  67. // Pre-upload action name
  68. OperationPreUpload = "pre-upload"
  69. operationPreDelete = "pre-delete"
  70. operationRename = "rename"
  71. operationMkdir = "mkdir"
  72. operationRmdir = "rmdir"
  73. // SSH command action name
  74. OperationSSHCmd = "ssh_cmd"
  75. chtimesFormat = "2006-01-02T15:04:05" // YYYY-MM-DDTHH:MM:SS
  76. idleTimeoutCheckInterval = 3 * time.Minute
  77. periodicTimeoutCheckInterval = 1 * time.Minute
  78. )
  79. // Stat flags
  80. const (
  81. StatAttrUIDGID = 1
  82. StatAttrPerms = 2
  83. StatAttrTimes = 4
  84. StatAttrSize = 8
  85. )
  86. // Transfer types
  87. const (
  88. TransferUpload = iota
  89. TransferDownload
  90. )
  91. // Supported protocols
  92. const (
  93. ProtocolSFTP = "SFTP"
  94. ProtocolSCP = "SCP"
  95. ProtocolSSH = "SSH"
  96. ProtocolFTP = "FTP"
  97. ProtocolWebDAV = "DAV"
  98. ProtocolHTTP = "HTTP"
  99. ProtocolHTTPShare = "HTTPShare"
  100. ProtocolDataRetention = "DataRetention"
  101. ProtocolOIDC = "OIDC"
  102. protocolEventAction = "EventAction"
  103. )
  104. // Upload modes
  105. const (
  106. UploadModeStandard = 0
  107. UploadModeAtomic = 1
  108. UploadModeAtomicWithResume = 2
  109. UploadModeS3StoreOnError = 4
  110. UploadModeGCSStoreOnError = 8
  111. UploadModeAzureBlobStoreOnError = 16
  112. )
  113. func init() {
  114. Connections.clients = clientsMap{
  115. clients: make(map[string]int),
  116. }
  117. Connections.perUserConns = make(map[string]int)
  118. Connections.mapping = make(map[string]int)
  119. Connections.sshMapping = make(map[string]int)
  120. }
  121. // errors definitions
  122. var (
  123. ErrPermissionDenied = errors.New("permission denied")
  124. ErrNotExist = errors.New("no such file or directory")
  125. ErrOpUnsupported = errors.New("operation unsupported")
  126. ErrGenericFailure = errors.New("failure")
  127. ErrQuotaExceeded = errors.New("denying write due to space limit")
  128. ErrReadQuotaExceeded = errors.New("denying read due to quota limit")
  129. ErrConnectionDenied = errors.New("you are not allowed to connect")
  130. ErrNoBinding = errors.New("no binding configured")
  131. ErrCrtRevoked = errors.New("your certificate has been revoked")
  132. ErrNoCredentials = errors.New("no credential provided")
  133. ErrInternalFailure = errors.New("internal failure")
  134. ErrTransferAborted = errors.New("transfer aborted")
  135. ErrShuttingDown = errors.New("the service is shutting down")
  136. errNoTransfer = errors.New("requested transfer not found")
  137. errTransferMismatch = errors.New("transfer mismatch")
  138. )
  139. var (
  140. // Config is the configuration for the supported protocols
  141. Config Configuration
  142. // Connections is the list of active connections
  143. Connections ActiveConnections
  144. // QuotaScans is the list of active quota scans
  145. QuotaScans ActiveScans
  146. transfersChecker TransfersChecker
  147. supportedProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP, ProtocolWebDAV,
  148. ProtocolHTTP, ProtocolHTTPShare, ProtocolOIDC}
  149. disconnHookProtocols = []string{ProtocolSFTP, ProtocolSCP, ProtocolSSH, ProtocolFTP}
  150. // the map key is the protocol, for each protocol we can have multiple rate limiters
  151. rateLimiters map[string][]*rateLimiter
  152. isShuttingDown atomic.Bool
  153. ftpLoginCommands = []string{"PASS", "USER"}
  154. )
  155. // Initialize sets the common configuration
  156. func Initialize(c Configuration, isShared int) error {
  157. isShuttingDown.Store(false)
  158. util.SetUmask(c.Umask)
  159. version.SetConfig(c.ServerVersion)
  160. Config = c
  161. Config.Actions.ExecuteOn = util.RemoveDuplicates(Config.Actions.ExecuteOn, true)
  162. Config.Actions.ExecuteSync = util.RemoveDuplicates(Config.Actions.ExecuteSync, true)
  163. Config.ProxyAllowed = util.RemoveDuplicates(Config.ProxyAllowed, true)
  164. Config.idleLoginTimeout = 2 * time.Minute
  165. Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute
  166. startPeriodicChecks(periodicTimeoutCheckInterval, isShared)
  167. Config.defender = nil
  168. Config.allowList = nil
  169. Config.rateLimitersList = nil
  170. rateLimiters = make(map[string][]*rateLimiter)
  171. for _, rlCfg := range c.RateLimitersConfig {
  172. if rlCfg.isEnabled() {
  173. if err := rlCfg.validate(); err != nil {
  174. return fmt.Errorf("rate limiters initialization error: %w", err)
  175. }
  176. rateLimiter := rlCfg.getLimiter()
  177. for _, protocol := range rlCfg.Protocols {
  178. rateLimiters[protocol] = append(rateLimiters[protocol], rateLimiter)
  179. }
  180. }
  181. }
  182. if len(rateLimiters) > 0 {
  183. rateLimitersList, err := dataprovider.NewIPList(dataprovider.IPListTypeRateLimiterSafeList)
  184. if err != nil {
  185. return fmt.Errorf("unable to initialize ratelimiters list: %w", err)
  186. }
  187. Config.rateLimitersList = rateLimitersList
  188. }
  189. if c.DefenderConfig.Enabled {
  190. if !util.Contains(supportedDefenderDrivers, c.DefenderConfig.Driver) {
  191. return fmt.Errorf("unsupported defender driver %q", c.DefenderConfig.Driver)
  192. }
  193. var defender Defender
  194. var err error
  195. switch c.DefenderConfig.Driver {
  196. case DefenderDriverProvider:
  197. defender, err = newDBDefender(&c.DefenderConfig)
  198. default:
  199. defender, err = newInMemoryDefender(&c.DefenderConfig)
  200. }
  201. if err != nil {
  202. return fmt.Errorf("defender initialization error: %v", err)
  203. }
  204. logger.Info(logSender, "", "defender initialized with config %+v", c.DefenderConfig)
  205. Config.defender = defender
  206. }
  207. if c.AllowListStatus > 0 {
  208. allowList, err := dataprovider.NewIPList(dataprovider.IPListTypeAllowList)
  209. if err != nil {
  210. return fmt.Errorf("unable to initialize the allow list: %w", err)
  211. }
  212. logger.Info(logSender, "", "allow list initialized")
  213. Config.allowList = allowList
  214. }
  215. if err := c.initializeProxyProtocol(); err != nil {
  216. return err
  217. }
  218. vfs.SetTempPath(c.TempPath)
  219. dataprovider.SetTempPath(c.TempPath)
  220. vfs.SetAllowSelfConnections(c.AllowSelfConnections)
  221. vfs.SetRenameMode(c.RenameMode)
  222. vfs.SetReadMetadataMode(c.Metadata.Read)
  223. vfs.SetResumeMaxSize(c.ResumeMaxSize)
  224. vfs.SetUploadMode(c.UploadMode)
  225. dataprovider.SetAllowSelfConnections(c.AllowSelfConnections)
  226. transfersChecker = getTransfersChecker(isShared)
  227. return nil
  228. }
  229. // CheckClosing returns an error if the service is closing
  230. func CheckClosing() error {
  231. if isShuttingDown.Load() {
  232. return ErrShuttingDown
  233. }
  234. return nil
  235. }
  236. // WaitForTransfers waits, for the specified grace time, for currently ongoing
  237. // client-initiated transfer sessions to completes.
  238. // A zero graceTime means no wait
  239. func WaitForTransfers(graceTime int) {
  240. if graceTime == 0 {
  241. return
  242. }
  243. if isShuttingDown.Swap(true) {
  244. return
  245. }
  246. if activeHooks.Load() == 0 && getActiveConnections() == 0 {
  247. return
  248. }
  249. graceTimer := time.NewTimer(time.Duration(graceTime) * time.Second)
  250. ticker := time.NewTicker(3 * time.Second)
  251. for {
  252. select {
  253. case <-ticker.C:
  254. hooks := activeHooks.Load()
  255. logger.Info(logSender, "", "active hooks: %d", hooks)
  256. if hooks == 0 && getActiveConnections() == 0 {
  257. logger.Info(logSender, "", "no more active connections, graceful shutdown")
  258. ticker.Stop()
  259. graceTimer.Stop()
  260. return
  261. }
  262. case <-graceTimer.C:
  263. logger.Info(logSender, "", "grace time expired, hard shutdown")
  264. ticker.Stop()
  265. return
  266. }
  267. }
  268. }
  269. // getActiveConnections returns the number of connections with active transfers
  270. func getActiveConnections() int {
  271. var activeConns int
  272. Connections.RLock()
  273. for _, c := range Connections.connections {
  274. if len(c.GetTransfers()) > 0 {
  275. activeConns++
  276. }
  277. }
  278. Connections.RUnlock()
  279. logger.Info(logSender, "", "number of connections with active transfers: %d", activeConns)
  280. return activeConns
  281. }
  282. // LimitRate blocks until all the configured rate limiters
  283. // allow one event to happen.
  284. // It returns an error if the time to wait exceeds the max
  285. // allowed delay
  286. func LimitRate(protocol, ip string) (time.Duration, error) {
  287. if Config.rateLimitersList != nil {
  288. isListed, _, err := Config.rateLimitersList.IsListed(ip, protocol)
  289. if err == nil && isListed {
  290. return 0, nil
  291. }
  292. }
  293. for _, limiter := range rateLimiters[protocol] {
  294. if delay, err := limiter.Wait(ip, protocol); err != nil {
  295. logger.Debug(logSender, "", "protocol %s ip %s: %v", protocol, ip, err)
  296. return delay, err
  297. }
  298. }
  299. return 0, nil
  300. }
  301. // Reload reloads the whitelist, the IP filter plugin and the defender's block and safe lists
  302. func Reload() error {
  303. plugin.Handler.ReloadFilter()
  304. return nil
  305. }
  306. // IsBanned returns true if the specified IP address is banned
  307. func IsBanned(ip, protocol string) bool {
  308. if plugin.Handler.IsIPBanned(ip, protocol) {
  309. return true
  310. }
  311. if Config.defender == nil {
  312. return false
  313. }
  314. return Config.defender.IsBanned(ip, protocol)
  315. }
  316. // GetDefenderBanTime returns the ban time for the given IP
  317. // or nil if the IP is not banned or the defender is disabled
  318. func GetDefenderBanTime(ip string) (*time.Time, error) {
  319. if Config.defender == nil {
  320. return nil, nil
  321. }
  322. return Config.defender.GetBanTime(ip)
  323. }
  324. // GetDefenderHosts returns hosts that are banned or for which some violations have been detected
  325. func GetDefenderHosts() ([]dataprovider.DefenderEntry, error) {
  326. if Config.defender == nil {
  327. return nil, nil
  328. }
  329. return Config.defender.GetHosts()
  330. }
  331. // GetDefenderHost returns a defender host by ip, if any
  332. func GetDefenderHost(ip string) (dataprovider.DefenderEntry, error) {
  333. if Config.defender == nil {
  334. return dataprovider.DefenderEntry{}, errors.New("defender is disabled")
  335. }
  336. return Config.defender.GetHost(ip)
  337. }
  338. // DeleteDefenderHost removes the specified IP address from the defender lists
  339. func DeleteDefenderHost(ip string) bool {
  340. if Config.defender == nil {
  341. return false
  342. }
  343. return Config.defender.DeleteHost(ip)
  344. }
  345. // GetDefenderScore returns the score for the given IP
  346. func GetDefenderScore(ip string) (int, error) {
  347. if Config.defender == nil {
  348. return 0, nil
  349. }
  350. return Config.defender.GetScore(ip)
  351. }
  352. // AddDefenderEvent adds the specified defender event for the given IP.
  353. // Returns true if the IP is in the defender's safe list.
  354. func AddDefenderEvent(ip, protocol string, event HostEvent) bool {
  355. if Config.defender == nil {
  356. return false
  357. }
  358. return Config.defender.AddEvent(ip, protocol, event)
  359. }
  360. func startPeriodicChecks(duration time.Duration, isShared int) {
  361. startEventScheduler()
  362. spec := fmt.Sprintf("@every %s", duration)
  363. _, err := eventScheduler.AddFunc(spec, Connections.checkTransfers)
  364. util.PanicOnError(err)
  365. logger.Info(logSender, "", "scheduled overquota transfers check, schedule %q", spec)
  366. if isShared == 1 {
  367. logger.Info(logSender, "", "add reload configs task")
  368. _, err := eventScheduler.AddFunc("@every 10m", smtp.ReloadProviderConf)
  369. util.PanicOnError(err)
  370. }
  371. if Config.IdleTimeout > 0 {
  372. ratio := idleTimeoutCheckInterval / periodicTimeoutCheckInterval
  373. spec = fmt.Sprintf("@every %s", duration*ratio)
  374. _, err = eventScheduler.AddFunc(spec, Connections.checkIdles)
  375. util.PanicOnError(err)
  376. logger.Info(logSender, "", "scheduled idle connections check, schedule %q", spec)
  377. }
  378. }
  379. // ActiveTransfer defines the interface for the current active transfers
  380. type ActiveTransfer interface {
  381. GetID() int64
  382. GetType() int
  383. GetSize() int64
  384. GetDownloadedSize() int64
  385. GetUploadedSize() int64
  386. GetVirtualPath() string
  387. GetStartTime() time.Time
  388. SignalClose(err error)
  389. Truncate(fsPath string, size int64) (int64, error)
  390. GetRealFsPath(fsPath string) string
  391. SetTimes(fsPath string, atime time.Time, mtime time.Time) bool
  392. GetTruncatedSize() int64
  393. HasSizeLimit() bool
  394. }
  395. // ActiveConnection defines the interface for the current active connections
  396. type ActiveConnection interface {
  397. GetID() string
  398. GetUsername() string
  399. GetRole() string
  400. GetMaxSessions() int
  401. GetLocalAddress() string
  402. GetRemoteAddress() string
  403. GetClientVersion() string
  404. GetProtocol() string
  405. GetConnectionTime() time.Time
  406. GetLastActivity() time.Time
  407. GetCommand() string
  408. Disconnect() error
  409. AddTransfer(t ActiveTransfer)
  410. RemoveTransfer(t ActiveTransfer)
  411. GetTransfers() []ConnectionTransfer
  412. SignalTransferClose(transferID int64, err error)
  413. CloseFS() error
  414. isAccessAllowed() bool
  415. }
  416. // StatAttributes defines the attributes for set stat commands
  417. type StatAttributes struct {
  418. Mode os.FileMode
  419. Atime time.Time
  420. Mtime time.Time
  421. UID int
  422. GID int
  423. Flags int
  424. Size int64
  425. }
  426. // ConnectionTransfer defines the trasfer details
  427. type ConnectionTransfer struct {
  428. ID int64 `json:"-"`
  429. OperationType string `json:"operation_type"`
  430. StartTime int64 `json:"start_time"`
  431. Size int64 `json:"size"`
  432. VirtualPath string `json:"path"`
  433. HasSizeLimit bool `json:"-"`
  434. ULSize int64 `json:"-"`
  435. DLSize int64 `json:"-"`
  436. }
  437. // MetadataConfig defines how to handle metadata for cloud storage backends
  438. type MetadataConfig struct {
  439. // If not zero the metadata will be read before downloads and will be
  440. // available in notifications
  441. Read int `json:"read" mapstructure:"read"`
  442. }
  443. // Configuration defines configuration parameters common to all supported protocols
  444. type Configuration struct {
  445. // Maximum idle timeout as minutes. If a client is idle for a time that exceeds this setting it will be disconnected.
  446. // 0 means disabled
  447. IdleTimeout int `json:"idle_timeout" mapstructure:"idle_timeout"`
  448. // UploadMode 0 means standard, the files are uploaded directly to the requested path.
  449. // 1 means atomic: the files are uploaded to a temporary path and renamed to the requested path
  450. // when the client ends the upload. Atomic mode avoid problems such as a web server that
  451. // serves partial files when the files are being uploaded.
  452. // In atomic mode if there is an upload error the temporary file is deleted and so the requested
  453. // upload path will not contain a partial file.
  454. // 2 means atomic with resume support: as atomic but if there is an upload error the temporary
  455. // file is renamed to the requested path and not deleted, this way a client can reconnect and resume
  456. // the upload.
  457. // 4 means files for S3 backend are stored even if a client-side upload error is detected.
  458. // 8 means files for Google Cloud Storage backend are stored even if a client-side upload error is detected.
  459. // 16 means files for Azure Blob backend are stored even if a client-side upload error is detected.
  460. UploadMode int `json:"upload_mode" mapstructure:"upload_mode"`
  461. // Actions to execute for SFTP file operations and SSH commands
  462. Actions ProtocolActions `json:"actions" mapstructure:"actions"`
  463. // SetstatMode 0 means "normal mode": requests for changing permissions and owner/group are executed.
  464. // 1 means "ignore mode": requests for changing permissions and owner/group are silently ignored.
  465. // 2 means "ignore mode for cloud fs": requests for changing permissions and owner/group are
  466. // silently ignored for cloud based filesystem such as S3, GCS, Azure Blob. Requests for changing
  467. // modification times are ignored for cloud based filesystem if they are not supported.
  468. SetstatMode int `json:"setstat_mode" mapstructure:"setstat_mode"`
  469. // RenameMode defines how to handle directory renames. By default, renaming of non-empty directories
  470. // is not allowed for cloud storage providers (S3, GCS, Azure Blob). Set to 1 to enable recursive
  471. // renames for these providers, they may be slow, there is no atomic rename API like for local
  472. // filesystem, so SFTPGo will recursively list the directory contents and do a rename for each entry
  473. RenameMode int `json:"rename_mode" mapstructure:"rename_mode"`
  474. // ResumeMaxSize defines the maximum size allowed, in bytes, to resume uploads on storage backends
  475. // with immutable objects. By default, resuming uploads is not allowed for cloud storage providers
  476. // (S3, GCS, Azure Blob) because SFTPGo must rewrite the entire file.
  477. // Set to a value greater than 0 to allow resuming uploads of files smaller than or equal to the
  478. // defined size.
  479. ResumeMaxSize int64 `json:"resume_max_size" mapstructure:"resume_max_size"`
  480. // TempPath defines the path for temporary files such as those used for atomic uploads or file pipes.
  481. // If you set this option you must make sure that the defined path exists, is accessible for writing
  482. // by the user running SFTPGo, and is on the same filesystem as the users home directories otherwise
  483. // the renaming for atomic uploads will become a copy and therefore may take a long time.
  484. // The temporary files are not namespaced. The default is generally fine. Leave empty for the default.
  485. TempPath string `json:"temp_path" mapstructure:"temp_path"`
  486. // Support for HAProxy PROXY protocol.
  487. // If you are running SFTPGo behind a proxy server such as HAProxy, AWS ELB or NGNIX, you can enable
  488. // the proxy protocol. It provides a convenient way to safely transport connection information
  489. // such as a client's address across multiple layers of NAT or TCP proxies to get the real
  490. // client IP address instead of the proxy IP. Both protocol versions 1 and 2 are supported.
  491. // - 0 means disabled
  492. // - 1 means proxy protocol enabled. Proxy header will be used and requests without proxy header will be accepted.
  493. // - 2 means proxy protocol required. Proxy header will be used and requests without proxy header will be rejected.
  494. // If the proxy protocol is enabled in SFTPGo then you have to enable the protocol in your proxy configuration too,
  495. // for example for HAProxy add "send-proxy" or "send-proxy-v2" to each server configuration line.
  496. ProxyProtocol int `json:"proxy_protocol" mapstructure:"proxy_protocol"`
  497. // List of IP addresses and IP ranges allowed to send the proxy header.
  498. // If proxy protocol is set to 1 and we receive a proxy header from an IP that is not in the list then the
  499. // connection will be accepted and the header will be ignored.
  500. // If proxy protocol is set to 2 and we receive a proxy header from an IP that is not in the list then the
  501. // connection will be rejected.
  502. ProxyAllowed []string `json:"proxy_allowed" mapstructure:"proxy_allowed"`
  503. // List of IP addresses and IP ranges for which not to read the proxy header
  504. ProxySkipped []string `json:"proxy_skipped" mapstructure:"proxy_skipped"`
  505. // Absolute path to an external program or an HTTP URL to invoke as soon as SFTPGo starts.
  506. // If you define an HTTP URL it will be invoked using a `GET` request.
  507. // Please note that SFTPGo services may not yet be available when this hook is run.
  508. // Leave empty do disable.
  509. StartupHook string `json:"startup_hook" mapstructure:"startup_hook"`
  510. // Absolute path to an external program or an HTTP URL to invoke after a user connects
  511. // and before he tries to login. It allows you to reject the connection based on the source
  512. // ip address. Leave empty do disable.
  513. PostConnectHook string `json:"post_connect_hook" mapstructure:"post_connect_hook"`
  514. // Absolute path to an external program or an HTTP URL to invoke after an SSH/FTP connection ends.
  515. // Leave empty do disable.
  516. PostDisconnectHook string `json:"post_disconnect_hook" mapstructure:"post_disconnect_hook"`
  517. // Absolute path to an external program or an HTTP URL to invoke after a data retention check completes.
  518. // Leave empty do disable.
  519. DataRetentionHook string `json:"data_retention_hook" mapstructure:"data_retention_hook"`
  520. // Maximum number of concurrent client connections. 0 means unlimited
  521. MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"`
  522. // Maximum number of concurrent client connections from the same host (IP). 0 means unlimited
  523. MaxPerHostConnections int `json:"max_per_host_connections" mapstructure:"max_per_host_connections"`
  524. // Defines the status of the global allow list. 0 means disabled, 1 enabled.
  525. // If enabled, only the listed IPs/networks can access the configured services, all other
  526. // client connections will be dropped before they even try to authenticate.
  527. // Ensure to enable this setting only after adding some allowed ip/networks from the WebAdmin/REST API
  528. AllowListStatus int `json:"allowlist_status" mapstructure:"allowlist_status"`
  529. // Allow users on this instance to use other users/virtual folders on this instance as storage backend.
  530. // Enable this setting if you know what you are doing.
  531. AllowSelfConnections int `json:"allow_self_connections" mapstructure:"allow_self_connections"`
  532. // Defender configuration
  533. DefenderConfig DefenderConfig `json:"defender" mapstructure:"defender"`
  534. // Rate limiter configurations
  535. RateLimitersConfig []RateLimiterConfig `json:"rate_limiters" mapstructure:"rate_limiters"`
  536. // Umask for new uploads. Leave blank to use the system default.
  537. Umask string `json:"umask" mapstructure:"umask"`
  538. // Defines the server version
  539. ServerVersion string `json:"server_version" mapstructure:"server_version"`
  540. // Metadata configuration
  541. Metadata MetadataConfig `json:"metadata" mapstructure:"metadata"`
  542. idleTimeoutAsDuration time.Duration
  543. idleLoginTimeout time.Duration
  544. defender Defender
  545. allowList *dataprovider.IPList
  546. rateLimitersList *dataprovider.IPList
  547. proxyAllowed []func(net.IP) bool
  548. proxySkipped []func(net.IP) bool
  549. }
  550. // IsAtomicUploadEnabled returns true if atomic upload is enabled
  551. func (c *Configuration) IsAtomicUploadEnabled() bool {
  552. return c.UploadMode&UploadModeAtomic != 0 || c.UploadMode&UploadModeAtomicWithResume != 0
  553. }
  554. func (c *Configuration) initializeProxyProtocol() error {
  555. if c.ProxyProtocol > 0 {
  556. allowed, err := util.ParseAllowedIPAndRanges(c.ProxyAllowed)
  557. if err != nil {
  558. return fmt.Errorf("invalid proxy allowed: %w", err)
  559. }
  560. skipped, err := util.ParseAllowedIPAndRanges(c.ProxySkipped)
  561. if err != nil {
  562. return fmt.Errorf("invalid proxy skipped: %w", err)
  563. }
  564. Config.proxyAllowed = allowed
  565. Config.proxySkipped = skipped
  566. }
  567. return nil
  568. }
  569. // GetProxyListener returns a wrapper for the given listener that supports the
  570. // HAProxy Proxy Protocol
  571. func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Listener, error) {
  572. if c.ProxyProtocol > 0 {
  573. defaultPolicy := proxyproto.REQUIRE
  574. if c.ProxyProtocol == 1 {
  575. defaultPolicy = proxyproto.IGNORE
  576. }
  577. return &proxyproto.Listener{
  578. Listener: listener,
  579. Policy: getProxyPolicy(c.proxyAllowed, c.proxySkipped, defaultPolicy),
  580. ReadHeaderTimeout: 10 * time.Second,
  581. }, nil
  582. }
  583. return nil, errors.New("proxy protocol not configured")
  584. }
  585. // GetRateLimitersStatus returns the rate limiters status
  586. func (c *Configuration) GetRateLimitersStatus() (bool, []string) {
  587. enabled := false
  588. var protocols []string
  589. for _, rlCfg := range c.RateLimitersConfig {
  590. if rlCfg.isEnabled() {
  591. enabled = true
  592. protocols = append(protocols, rlCfg.Protocols...)
  593. }
  594. }
  595. return enabled, util.RemoveDuplicates(protocols, false)
  596. }
  597. // IsAllowListEnabled returns true if the global allow list is enabled
  598. func (c *Configuration) IsAllowListEnabled() bool {
  599. return c.AllowListStatus > 0
  600. }
  601. // ExecuteStartupHook runs the startup hook if defined
  602. func (c *Configuration) ExecuteStartupHook() error {
  603. if c.StartupHook == "" {
  604. return nil
  605. }
  606. if strings.HasPrefix(c.StartupHook, "http") {
  607. var url *url.URL
  608. url, err := url.Parse(c.StartupHook)
  609. if err != nil {
  610. logger.Warn(logSender, "", "Invalid startup hook %q: %v", c.StartupHook, err)
  611. return err
  612. }
  613. startTime := time.Now()
  614. resp, err := httpclient.RetryableGet(url.String())
  615. if err != nil {
  616. logger.Warn(logSender, "", "Error executing startup hook: %v", err)
  617. return err
  618. }
  619. defer resp.Body.Close()
  620. logger.Debug(logSender, "", "Startup hook executed, elapsed: %v, response code: %v", time.Since(startTime), resp.StatusCode)
  621. return nil
  622. }
  623. if !filepath.IsAbs(c.StartupHook) {
  624. err := fmt.Errorf("invalid startup hook %q", c.StartupHook)
  625. logger.Warn(logSender, "", "Invalid startup hook %q", c.StartupHook)
  626. return err
  627. }
  628. startTime := time.Now()
  629. timeout, env, args := command.GetConfig(c.StartupHook, command.HookStartup)
  630. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  631. defer cancel()
  632. cmd := exec.CommandContext(ctx, c.StartupHook, args...)
  633. cmd.Env = env
  634. err := cmd.Run()
  635. logger.Debug(logSender, "", "Startup hook executed, elapsed: %s, error: %v", time.Since(startTime), err)
  636. return nil
  637. }
  638. func (c *Configuration) executePostDisconnectHook(remoteAddr, protocol, username, connID string, connectionTime time.Time) {
  639. startNewHook()
  640. defer hookEnded()
  641. ipAddr := util.GetIPFromRemoteAddress(remoteAddr)
  642. connDuration := int64(time.Since(connectionTime) / time.Millisecond)
  643. if strings.HasPrefix(c.PostDisconnectHook, "http") {
  644. var url *url.URL
  645. url, err := url.Parse(c.PostDisconnectHook)
  646. if err != nil {
  647. logger.Warn(protocol, connID, "Invalid post disconnect hook %q: %v", c.PostDisconnectHook, err)
  648. return
  649. }
  650. q := url.Query()
  651. q.Add("ip", ipAddr)
  652. q.Add("protocol", protocol)
  653. q.Add("username", username)
  654. q.Add("connection_duration", strconv.FormatInt(connDuration, 10))
  655. url.RawQuery = q.Encode()
  656. startTime := time.Now()
  657. resp, err := httpclient.RetryableGet(url.String())
  658. respCode := 0
  659. if err == nil {
  660. respCode = resp.StatusCode
  661. resp.Body.Close()
  662. }
  663. logger.Debug(protocol, connID, "Post disconnect hook response code: %v, elapsed: %v, err: %v",
  664. respCode, time.Since(startTime), err)
  665. return
  666. }
  667. if !filepath.IsAbs(c.PostDisconnectHook) {
  668. logger.Debug(protocol, connID, "invalid post disconnect hook %q", c.PostDisconnectHook)
  669. return
  670. }
  671. timeout, env, args := command.GetConfig(c.PostDisconnectHook, command.HookPostDisconnect)
  672. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  673. defer cancel()
  674. startTime := time.Now()
  675. cmd := exec.CommandContext(ctx, c.PostDisconnectHook, args...)
  676. cmd.Env = append(env,
  677. fmt.Sprintf("SFTPGO_CONNECTION_IP=%s", ipAddr),
  678. fmt.Sprintf("SFTPGO_CONNECTION_USERNAME=%s", username),
  679. fmt.Sprintf("SFTPGO_CONNECTION_DURATION=%d", connDuration),
  680. fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%s", protocol))
  681. err := cmd.Run()
  682. logger.Debug(protocol, connID, "Post disconnect hook executed, elapsed: %s error: %v", time.Since(startTime), err)
  683. }
  684. func (c *Configuration) checkPostDisconnectHook(remoteAddr, protocol, username, connID string, connectionTime time.Time) {
  685. if c.PostDisconnectHook == "" {
  686. return
  687. }
  688. if !util.Contains(disconnHookProtocols, protocol) {
  689. return
  690. }
  691. go c.executePostDisconnectHook(remoteAddr, protocol, username, connID, connectionTime)
  692. }
  693. // ExecutePostConnectHook executes the post connect hook if defined
  694. func (c *Configuration) ExecutePostConnectHook(ipAddr, protocol string) error {
  695. if c.PostConnectHook == "" {
  696. return nil
  697. }
  698. if strings.HasPrefix(c.PostConnectHook, "http") {
  699. var url *url.URL
  700. url, err := url.Parse(c.PostConnectHook)
  701. if err != nil {
  702. logger.Warn(protocol, "", "Login from ip %q denied, invalid post connect hook %q: %v",
  703. ipAddr, c.PostConnectHook, err)
  704. return getPermissionDeniedError(protocol)
  705. }
  706. q := url.Query()
  707. q.Add("ip", ipAddr)
  708. q.Add("protocol", protocol)
  709. url.RawQuery = q.Encode()
  710. resp, err := httpclient.RetryableGet(url.String())
  711. if err != nil {
  712. logger.Warn(protocol, "", "Login from ip %q denied, error executing post connect hook: %v", ipAddr, err)
  713. return getPermissionDeniedError(protocol)
  714. }
  715. defer resp.Body.Close()
  716. if resp.StatusCode != http.StatusOK {
  717. logger.Warn(protocol, "", "Login from ip %q denied, post connect hook response code: %v", ipAddr, resp.StatusCode)
  718. return getPermissionDeniedError(protocol)
  719. }
  720. return nil
  721. }
  722. if !filepath.IsAbs(c.PostConnectHook) {
  723. err := fmt.Errorf("invalid post connect hook %q", c.PostConnectHook)
  724. logger.Warn(protocol, "", "Login from ip %q denied: %v", ipAddr, err)
  725. return getPermissionDeniedError(protocol)
  726. }
  727. timeout, env, args := command.GetConfig(c.PostConnectHook, command.HookPostConnect)
  728. ctx, cancel := context.WithTimeout(context.Background(), timeout)
  729. defer cancel()
  730. cmd := exec.CommandContext(ctx, c.PostConnectHook, args...)
  731. cmd.Env = append(env,
  732. fmt.Sprintf("SFTPGO_CONNECTION_IP=%s", ipAddr),
  733. fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%s", protocol))
  734. err := cmd.Run()
  735. if err != nil {
  736. logger.Warn(protocol, "", "Login from ip %q denied, connect hook error: %v", ipAddr, err)
  737. return getPermissionDeniedError(protocol)
  738. }
  739. return nil
  740. }
  741. func getProxyPolicy(allowed, skipped []func(net.IP) bool, def proxyproto.Policy) proxyproto.PolicyFunc {
  742. return func(upstream net.Addr) (proxyproto.Policy, error) {
  743. upstreamIP, err := util.GetIPFromNetAddr(upstream)
  744. if err != nil {
  745. // something is wrong with the source IP, better reject the connection
  746. return proxyproto.REJECT, err
  747. }
  748. for _, skippedFrom := range skipped {
  749. if skippedFrom(upstreamIP) {
  750. return proxyproto.SKIP, nil
  751. }
  752. }
  753. for _, allowFrom := range allowed {
  754. if allowFrom(upstreamIP) {
  755. if def == proxyproto.REQUIRE {
  756. return proxyproto.REQUIRE, nil
  757. }
  758. return proxyproto.USE, nil
  759. }
  760. }
  761. return def, nil
  762. }
  763. }
  764. // SSHConnection defines an ssh connection.
  765. // Each SSH connection can open several channels for SFTP or SSH commands
  766. type SSHConnection struct {
  767. id string
  768. conn net.Conn
  769. lastActivity atomic.Int64
  770. }
  771. // NewSSHConnection returns a new SSHConnection
  772. func NewSSHConnection(id string, conn net.Conn) *SSHConnection {
  773. c := &SSHConnection{
  774. id: id,
  775. conn: conn,
  776. }
  777. c.lastActivity.Store(time.Now().UnixNano())
  778. return c
  779. }
  780. // GetID returns the ID for this SSHConnection
  781. func (c *SSHConnection) GetID() string {
  782. return c.id
  783. }
  784. // UpdateLastActivity updates last activity for this connection
  785. func (c *SSHConnection) UpdateLastActivity() {
  786. c.lastActivity.Store(time.Now().UnixNano())
  787. }
  788. // GetLastActivity returns the last connection activity
  789. func (c *SSHConnection) GetLastActivity() time.Time {
  790. return time.Unix(0, c.lastActivity.Load())
  791. }
  792. // Close closes the underlying network connection
  793. func (c *SSHConnection) Close() error {
  794. return c.conn.Close()
  795. }
  796. // ActiveConnections holds the currect active connections with the associated transfers
  797. type ActiveConnections struct {
  798. // clients contains both authenticated and estabilished connections and the ones waiting
  799. // for authentication
  800. clients clientsMap
  801. transfersCheckStatus atomic.Bool
  802. sync.RWMutex
  803. connections []ActiveConnection
  804. mapping map[string]int
  805. sshConnections []*SSHConnection
  806. sshMapping map[string]int
  807. perUserConns map[string]int
  808. }
  809. // internal method, must be called within a locked block
  810. func (conns *ActiveConnections) addUserConnection(username string) {
  811. if username == "" {
  812. return
  813. }
  814. conns.perUserConns[username]++
  815. }
  816. // internal method, must be called within a locked block
  817. func (conns *ActiveConnections) removeUserConnection(username string) {
  818. if username == "" {
  819. return
  820. }
  821. if val, ok := conns.perUserConns[username]; ok {
  822. conns.perUserConns[username]--
  823. if val > 1 {
  824. return
  825. }
  826. delete(conns.perUserConns, username)
  827. }
  828. }
  829. // GetActiveSessions returns the number of active sessions for the given username.
  830. // We return the open sessions for any protocol
  831. func (conns *ActiveConnections) GetActiveSessions(username string) int {
  832. conns.RLock()
  833. defer conns.RUnlock()
  834. return conns.perUserConns[username]
  835. }
  836. // Add adds a new connection to the active ones
  837. func (conns *ActiveConnections) Add(c ActiveConnection) error {
  838. conns.Lock()
  839. defer conns.Unlock()
  840. if username := c.GetUsername(); username != "" {
  841. if maxSessions := c.GetMaxSessions(); maxSessions > 0 {
  842. if val := conns.perUserConns[username]; val >= maxSessions {
  843. return fmt.Errorf("too many open sessions: %d/%d", val, maxSessions)
  844. }
  845. }
  846. conns.addUserConnection(username)
  847. }
  848. conns.mapping[c.GetID()] = len(conns.connections)
  849. conns.connections = append(conns.connections, c)
  850. metric.UpdateActiveConnectionsSize(len(conns.connections))
  851. logger.Debug(c.GetProtocol(), c.GetID(), "connection added, local address %q, remote address %q, num open connections: %d",
  852. c.GetLocalAddress(), c.GetRemoteAddress(), len(conns.connections))
  853. return nil
  854. }
  855. // Swap replaces an existing connection with the given one.
  856. // This method is useful if you have to change some connection details
  857. // for example for FTP is used to update the connection once the user
  858. // authenticates
  859. func (conns *ActiveConnections) Swap(c ActiveConnection) error {
  860. conns.Lock()
  861. defer conns.Unlock()
  862. if idx, ok := conns.mapping[c.GetID()]; ok {
  863. conn := conns.connections[idx]
  864. conns.removeUserConnection(conn.GetUsername())
  865. if username := c.GetUsername(); username != "" {
  866. if maxSessions := c.GetMaxSessions(); maxSessions > 0 {
  867. if val, ok := conns.perUserConns[username]; ok && val >= maxSessions {
  868. conns.addUserConnection(conn.GetUsername())
  869. return fmt.Errorf("too many open sessions: %d/%d", val, maxSessions)
  870. }
  871. }
  872. conns.addUserConnection(username)
  873. }
  874. err := conn.CloseFS()
  875. conns.connections[idx] = c
  876. logger.Debug(logSender, c.GetID(), "connection swapped, close fs error: %v", err)
  877. conn = nil
  878. return nil
  879. }
  880. return errors.New("connection to swap not found")
  881. }
  882. // Remove removes a connection from the active ones
  883. func (conns *ActiveConnections) Remove(connectionID string) {
  884. conns.Lock()
  885. defer conns.Unlock()
  886. if idx, ok := conns.mapping[connectionID]; ok {
  887. conn := conns.connections[idx]
  888. err := conn.CloseFS()
  889. lastIdx := len(conns.connections) - 1
  890. conns.connections[idx] = conns.connections[lastIdx]
  891. conns.connections[lastIdx] = nil
  892. conns.connections = conns.connections[:lastIdx]
  893. delete(conns.mapping, connectionID)
  894. if idx != lastIdx {
  895. conns.mapping[conns.connections[idx].GetID()] = idx
  896. }
  897. conns.removeUserConnection(conn.GetUsername())
  898. metric.UpdateActiveConnectionsSize(lastIdx)
  899. logger.Debug(conn.GetProtocol(), conn.GetID(), "connection removed, local address %q, remote address %q close fs error: %v, num open connections: %d",
  900. conn.GetLocalAddress(), conn.GetRemoteAddress(), err, lastIdx)
  901. if conn.GetProtocol() == ProtocolFTP && conn.GetUsername() == "" && !util.Contains(ftpLoginCommands, conn.GetCommand()) {
  902. ip := util.GetIPFromRemoteAddress(conn.GetRemoteAddress())
  903. logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTried, ProtocolFTP,
  904. dataprovider.ErrNoAuthTried.Error())
  905. metric.AddNoAuthTried()
  906. AddDefenderEvent(ip, ProtocolFTP, HostEventNoLoginTried)
  907. dataprovider.ExecutePostLoginHook(&dataprovider.User{}, dataprovider.LoginMethodNoAuthTried, ip,
  908. ProtocolFTP, dataprovider.ErrNoAuthTried)
  909. plugin.Handler.NotifyLogEvent(notifier.LogEventTypeNoLoginTried, ProtocolFTP, "", ip, "",
  910. dataprovider.ErrNoAuthTried)
  911. }
  912. Config.checkPostDisconnectHook(conn.GetRemoteAddress(), conn.GetProtocol(), conn.GetUsername(),
  913. conn.GetID(), conn.GetConnectionTime())
  914. return
  915. }
  916. logger.Debug(logSender, "", "connection id %q to remove not found!", connectionID)
  917. }
  918. // Close closes an active connection.
  919. // It returns true on success
  920. func (conns *ActiveConnections) Close(connectionID, role string) bool {
  921. conns.RLock()
  922. var result bool
  923. if idx, ok := conns.mapping[connectionID]; ok {
  924. c := conns.connections[idx]
  925. if role == "" || c.GetRole() == role {
  926. defer func(conn ActiveConnection) {
  927. err := conn.Disconnect()
  928. logger.Debug(conn.GetProtocol(), conn.GetID(), "close connection requested, close err: %v", err)
  929. }(c)
  930. result = true
  931. }
  932. }
  933. conns.RUnlock()
  934. return result
  935. }
  936. // AddSSHConnection adds a new ssh connection to the active ones
  937. func (conns *ActiveConnections) AddSSHConnection(c *SSHConnection) {
  938. conns.Lock()
  939. defer conns.Unlock()
  940. conns.sshMapping[c.GetID()] = len(conns.sshConnections)
  941. conns.sshConnections = append(conns.sshConnections, c)
  942. logger.Debug(logSender, c.GetID(), "ssh connection added, num open connections: %d", len(conns.sshConnections))
  943. }
  944. // RemoveSSHConnection removes a connection from the active ones
  945. func (conns *ActiveConnections) RemoveSSHConnection(connectionID string) {
  946. conns.Lock()
  947. defer conns.Unlock()
  948. if idx, ok := conns.sshMapping[connectionID]; ok {
  949. lastIdx := len(conns.sshConnections) - 1
  950. conns.sshConnections[idx] = conns.sshConnections[lastIdx]
  951. conns.sshConnections[lastIdx] = nil
  952. conns.sshConnections = conns.sshConnections[:lastIdx]
  953. delete(conns.sshMapping, connectionID)
  954. if idx != lastIdx {
  955. conns.sshMapping[conns.sshConnections[idx].GetID()] = idx
  956. }
  957. logger.Debug(logSender, connectionID, "ssh connection removed, num open ssh connections: %d", lastIdx)
  958. return
  959. }
  960. logger.Warn(logSender, "", "ssh connection to remove with id %q not found!", connectionID)
  961. }
  962. func (conns *ActiveConnections) checkIdles() {
  963. conns.RLock()
  964. for _, sshConn := range conns.sshConnections {
  965. idleTime := time.Since(sshConn.GetLastActivity())
  966. if idleTime > Config.idleTimeoutAsDuration {
  967. // we close an SSH connection if it has no active connections associated
  968. idToMatch := fmt.Sprintf("_%s_", sshConn.GetID())
  969. toClose := true
  970. for _, conn := range conns.connections {
  971. if strings.Contains(conn.GetID(), idToMatch) {
  972. if time.Since(conn.GetLastActivity()) <= Config.idleTimeoutAsDuration {
  973. toClose = false
  974. break
  975. }
  976. }
  977. }
  978. if toClose {
  979. defer func(c *SSHConnection) {
  980. err := c.Close()
  981. logger.Debug(logSender, c.GetID(), "close idle SSH connection, idle time: %v, close err: %v",
  982. time.Since(c.GetLastActivity()), err)
  983. }(sshConn)
  984. }
  985. }
  986. }
  987. for _, c := range conns.connections {
  988. idleTime := time.Since(c.GetLastActivity())
  989. isUnauthenticatedFTPUser := (c.GetProtocol() == ProtocolFTP && c.GetUsername() == "")
  990. if idleTime > Config.idleTimeoutAsDuration || (isUnauthenticatedFTPUser && idleTime > Config.idleLoginTimeout) {
  991. defer func(conn ActiveConnection) {
  992. err := conn.Disconnect()
  993. logger.Debug(conn.GetProtocol(), conn.GetID(), "close idle connection, idle time: %s, username: %q close err: %v",
  994. time.Since(conn.GetLastActivity()), conn.GetUsername(), err)
  995. }(c)
  996. } else if !c.isAccessAllowed() {
  997. defer func(conn ActiveConnection) {
  998. err := conn.Disconnect()
  999. logger.Info(conn.GetProtocol(), conn.GetID(), "access conditions not met for user: %q close connection err: %v",
  1000. conn.GetUsername(), err)
  1001. }(c)
  1002. }
  1003. }
  1004. conns.RUnlock()
  1005. }
  1006. func (conns *ActiveConnections) checkTransfers() {
  1007. if conns.transfersCheckStatus.Load() {
  1008. logger.Warn(logSender, "", "the previous transfer check is still running, skipping execution")
  1009. return
  1010. }
  1011. conns.transfersCheckStatus.Store(true)
  1012. defer conns.transfersCheckStatus.Store(false)
  1013. conns.RLock()
  1014. if len(conns.connections) < 2 {
  1015. conns.RUnlock()
  1016. return
  1017. }
  1018. var wg sync.WaitGroup
  1019. logger.Debug(logSender, "", "start concurrent transfers check")
  1020. // update the current size for transfers to monitors
  1021. for _, c := range conns.connections {
  1022. for _, t := range c.GetTransfers() {
  1023. if t.HasSizeLimit {
  1024. wg.Add(1)
  1025. go func(transfer ConnectionTransfer, connID string) {
  1026. defer wg.Done()
  1027. transfersChecker.UpdateTransferCurrentSizes(transfer.ULSize, transfer.DLSize, transfer.ID, connID)
  1028. }(t, c.GetID())
  1029. }
  1030. }
  1031. }
  1032. conns.RUnlock()
  1033. logger.Debug(logSender, "", "waiting for the update of the transfers current size")
  1034. wg.Wait()
  1035. logger.Debug(logSender, "", "getting overquota transfers")
  1036. overquotaTransfers := transfersChecker.GetOverquotaTransfers()
  1037. logger.Debug(logSender, "", "number of overquota transfers: %v", len(overquotaTransfers))
  1038. if len(overquotaTransfers) == 0 {
  1039. return
  1040. }
  1041. conns.RLock()
  1042. defer conns.RUnlock()
  1043. for _, c := range conns.connections {
  1044. for _, overquotaTransfer := range overquotaTransfers {
  1045. if c.GetID() == overquotaTransfer.ConnID {
  1046. logger.Info(logSender, c.GetID(), "user %q is overquota, try to close transfer id %v",
  1047. c.GetUsername(), overquotaTransfer.TransferID)
  1048. var err error
  1049. if overquotaTransfer.TransferType == TransferDownload {
  1050. err = getReadQuotaExceededError(c.GetProtocol())
  1051. } else {
  1052. err = getQuotaExceededError(c.GetProtocol())
  1053. }
  1054. c.SignalTransferClose(overquotaTransfer.TransferID, err)
  1055. }
  1056. }
  1057. }
  1058. logger.Debug(logSender, "", "transfers check completed")
  1059. }
  1060. // AddClientConnection stores a new client connection
  1061. func (conns *ActiveConnections) AddClientConnection(ipAddr string) {
  1062. conns.clients.add(ipAddr)
  1063. }
  1064. // RemoveClientConnection removes a disconnected client from the tracked ones
  1065. func (conns *ActiveConnections) RemoveClientConnection(ipAddr string) {
  1066. conns.clients.remove(ipAddr)
  1067. }
  1068. // GetClientConnections returns the total number of client connections
  1069. func (conns *ActiveConnections) GetClientConnections() int32 {
  1070. return conns.clients.getTotal()
  1071. }
  1072. // IsNewConnectionAllowed returns an error if the maximum number of concurrent allowed
  1073. // connections is exceeded or a whitelist is defined and the specified ipAddr is not listed
  1074. // or the service is shutting down
  1075. func (conns *ActiveConnections) IsNewConnectionAllowed(ipAddr, protocol string) error {
  1076. if isShuttingDown.Load() {
  1077. return ErrShuttingDown
  1078. }
  1079. if Config.allowList != nil {
  1080. isListed, _, err := Config.allowList.IsListed(ipAddr, protocol)
  1081. if err != nil {
  1082. logger.Error(logSender, "", "unable to query allow list, connection denied, ip %q, protocol %s, err: %v",
  1083. ipAddr, protocol, err)
  1084. return ErrConnectionDenied
  1085. }
  1086. if !isListed {
  1087. return ErrConnectionDenied
  1088. }
  1089. }
  1090. if Config.MaxTotalConnections == 0 && Config.MaxPerHostConnections == 0 {
  1091. return nil
  1092. }
  1093. if Config.MaxPerHostConnections > 0 {
  1094. if total := conns.clients.getTotalFrom(ipAddr); total > Config.MaxPerHostConnections {
  1095. if !AddDefenderEvent(ipAddr, protocol, HostEventLimitExceeded) {
  1096. logger.Warn(logSender, "", "connection denied, active connections from IP %q: %d/%d",
  1097. ipAddr, total, Config.MaxPerHostConnections)
  1098. return ErrConnectionDenied
  1099. }
  1100. logger.Info(logSender, "", "active connections from safe IP %q: %d", ipAddr, total)
  1101. }
  1102. }
  1103. if Config.MaxTotalConnections > 0 {
  1104. if total := conns.clients.getTotal(); total > int32(Config.MaxTotalConnections) {
  1105. logger.Info(logSender, "", "active client connections %d/%d", total, Config.MaxTotalConnections)
  1106. return ErrConnectionDenied
  1107. }
  1108. // on a single SFTP connection we could have multiple SFTP channels or commands
  1109. // so we check the estabilished connections too
  1110. conns.RLock()
  1111. defer conns.RUnlock()
  1112. if sess := len(conns.connections); sess >= Config.MaxTotalConnections {
  1113. logger.Info(logSender, "", "active client sessions %d/%d", sess, Config.MaxTotalConnections)
  1114. return ErrConnectionDenied
  1115. }
  1116. }
  1117. return nil
  1118. }
  1119. // GetStats returns stats for active connections
  1120. func (conns *ActiveConnections) GetStats(role string) []ConnectionStatus {
  1121. conns.RLock()
  1122. defer conns.RUnlock()
  1123. stats := make([]ConnectionStatus, 0, len(conns.connections))
  1124. node := dataprovider.GetNodeName()
  1125. for _, c := range conns.connections {
  1126. if role == "" || c.GetRole() == role {
  1127. stat := ConnectionStatus{
  1128. Username: c.GetUsername(),
  1129. ConnectionID: c.GetID(),
  1130. ClientVersion: c.GetClientVersion(),
  1131. RemoteAddress: c.GetRemoteAddress(),
  1132. ConnectionTime: util.GetTimeAsMsSinceEpoch(c.GetConnectionTime()),
  1133. LastActivity: util.GetTimeAsMsSinceEpoch(c.GetLastActivity()),
  1134. CurrentTime: util.GetTimeAsMsSinceEpoch(time.Now()),
  1135. Protocol: c.GetProtocol(),
  1136. Command: c.GetCommand(),
  1137. Transfers: c.GetTransfers(),
  1138. Node: node,
  1139. }
  1140. stats = append(stats, stat)
  1141. }
  1142. }
  1143. return stats
  1144. }
  1145. // ConnectionStatus returns the status for an active connection
  1146. type ConnectionStatus struct {
  1147. // Logged in username
  1148. Username string `json:"username"`
  1149. // Unique identifier for the connection
  1150. ConnectionID string `json:"connection_id"`
  1151. // client's version string
  1152. ClientVersion string `json:"client_version,omitempty"`
  1153. // Remote address for this connection
  1154. RemoteAddress string `json:"remote_address"`
  1155. // Connection time as unix timestamp in milliseconds
  1156. ConnectionTime int64 `json:"connection_time"`
  1157. // Last activity as unix timestamp in milliseconds
  1158. LastActivity int64 `json:"last_activity"`
  1159. // Current time as unix timestamp in milliseconds
  1160. CurrentTime int64 `json:"current_time"`
  1161. // Protocol for this connection
  1162. Protocol string `json:"protocol"`
  1163. // active uploads/downloads
  1164. Transfers []ConnectionTransfer `json:"active_transfers,omitempty"`
  1165. // SSH command or WebDAV method
  1166. Command string `json:"command,omitempty"`
  1167. // Node identifier, omitted for single node installations
  1168. Node string `json:"node,omitempty"`
  1169. }
  1170. // ActiveQuotaScan defines an active quota scan for a user
  1171. type ActiveQuotaScan struct {
  1172. // Username to which the quota scan refers
  1173. Username string `json:"username"`
  1174. // quota scan start time as unix timestamp in milliseconds
  1175. StartTime int64 `json:"start_time"`
  1176. Role string `json:"-"`
  1177. }
  1178. // ActiveVirtualFolderQuotaScan defines an active quota scan for a virtual folder
  1179. type ActiveVirtualFolderQuotaScan struct {
  1180. // folder name to which the quota scan refers
  1181. Name string `json:"name"`
  1182. // quota scan start time as unix timestamp in milliseconds
  1183. StartTime int64 `json:"start_time"`
  1184. }
  1185. // ActiveScans holds the active quota scans
  1186. type ActiveScans struct {
  1187. sync.RWMutex
  1188. UserScans []ActiveQuotaScan
  1189. FolderScans []ActiveVirtualFolderQuotaScan
  1190. }
  1191. // GetUsersQuotaScans returns the active users quota scans
  1192. func (s *ActiveScans) GetUsersQuotaScans(role string) []ActiveQuotaScan {
  1193. s.RLock()
  1194. defer s.RUnlock()
  1195. scans := make([]ActiveQuotaScan, 0, len(s.UserScans))
  1196. for _, scan := range s.UserScans {
  1197. if role == "" || role == scan.Role {
  1198. scans = append(scans, ActiveQuotaScan{
  1199. Username: scan.Username,
  1200. StartTime: scan.StartTime,
  1201. })
  1202. }
  1203. }
  1204. return scans
  1205. }
  1206. // AddUserQuotaScan adds a user to the ones with active quota scans.
  1207. // Returns false if the user has a quota scan already running
  1208. func (s *ActiveScans) AddUserQuotaScan(username, role string) bool {
  1209. s.Lock()
  1210. defer s.Unlock()
  1211. for _, scan := range s.UserScans {
  1212. if scan.Username == username {
  1213. return false
  1214. }
  1215. }
  1216. s.UserScans = append(s.UserScans, ActiveQuotaScan{
  1217. Username: username,
  1218. StartTime: util.GetTimeAsMsSinceEpoch(time.Now()),
  1219. Role: role,
  1220. })
  1221. return true
  1222. }
  1223. // RemoveUserQuotaScan removes a user from the ones with active quota scans.
  1224. // Returns false if the user has no active quota scans
  1225. func (s *ActiveScans) RemoveUserQuotaScan(username string) bool {
  1226. s.Lock()
  1227. defer s.Unlock()
  1228. for idx, scan := range s.UserScans {
  1229. if scan.Username == username {
  1230. lastIdx := len(s.UserScans) - 1
  1231. s.UserScans[idx] = s.UserScans[lastIdx]
  1232. s.UserScans = s.UserScans[:lastIdx]
  1233. return true
  1234. }
  1235. }
  1236. return false
  1237. }
  1238. // GetVFoldersQuotaScans returns the active quota scans for virtual folders
  1239. func (s *ActiveScans) GetVFoldersQuotaScans() []ActiveVirtualFolderQuotaScan {
  1240. s.RLock()
  1241. defer s.RUnlock()
  1242. scans := make([]ActiveVirtualFolderQuotaScan, len(s.FolderScans))
  1243. copy(scans, s.FolderScans)
  1244. return scans
  1245. }
  1246. // AddVFolderQuotaScan adds a virtual folder to the ones with active quota scans.
  1247. // Returns false if the folder has a quota scan already running
  1248. func (s *ActiveScans) AddVFolderQuotaScan(folderName string) bool {
  1249. s.Lock()
  1250. defer s.Unlock()
  1251. for _, scan := range s.FolderScans {
  1252. if scan.Name == folderName {
  1253. return false
  1254. }
  1255. }
  1256. s.FolderScans = append(s.FolderScans, ActiveVirtualFolderQuotaScan{
  1257. Name: folderName,
  1258. StartTime: util.GetTimeAsMsSinceEpoch(time.Now()),
  1259. })
  1260. return true
  1261. }
  1262. // RemoveVFolderQuotaScan removes a folder from the ones with active quota scans.
  1263. // Returns false if the folder has no active quota scans
  1264. func (s *ActiveScans) RemoveVFolderQuotaScan(folderName string) bool {
  1265. s.Lock()
  1266. defer s.Unlock()
  1267. for idx, scan := range s.FolderScans {
  1268. if scan.Name == folderName {
  1269. lastIdx := len(s.FolderScans) - 1
  1270. s.FolderScans[idx] = s.FolderScans[lastIdx]
  1271. s.FolderScans = s.FolderScans[:lastIdx]
  1272. return true
  1273. }
  1274. }
  1275. return false
  1276. }