httpfs.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810
  1. // Copyright (C) 2019-2023 Nicola Murino
  2. //
  3. // This program is free software: you can redistribute it and/or modify
  4. // it under the terms of the GNU Affero General Public License as published
  5. // by the Free Software Foundation, version 3.
  6. //
  7. // This program is distributed in the hope that it will be useful,
  8. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. // GNU Affero General Public License for more details.
  11. //
  12. // You should have received a copy of the GNU Affero General Public License
  13. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  14. package vfs
  15. import (
  16. "context"
  17. "crypto/tls"
  18. "encoding/json"
  19. "errors"
  20. "fmt"
  21. "io"
  22. "io/fs"
  23. "mime"
  24. "net"
  25. "net/http"
  26. "net/url"
  27. "os"
  28. "path"
  29. "path/filepath"
  30. "strings"
  31. "time"
  32. "github.com/eikenb/pipeat"
  33. "github.com/pkg/sftp"
  34. "github.com/sftpgo/sdk"
  35. "github.com/drakkan/sftpgo/v2/internal/kms"
  36. "github.com/drakkan/sftpgo/v2/internal/logger"
  37. "github.com/drakkan/sftpgo/v2/internal/metric"
  38. "github.com/drakkan/sftpgo/v2/internal/util"
  39. )
  40. const (
  41. // httpFsName is the name for the HTTP Fs implementation
  42. httpFsName = "httpfs"
  43. maxHTTPFsResponseSize = 1048576
  44. )
  45. var (
  46. supportedEndpointSchema = []string{"http://", "https://"}
  47. )
  48. // HTTPFsConfig defines the configuration for HTTP based filesystem
  49. type HTTPFsConfig struct {
  50. sdk.BaseHTTPFsConfig
  51. Password *kms.Secret `json:"password,omitempty"`
  52. APIKey *kms.Secret `json:"api_key,omitempty"`
  53. }
  54. func (c *HTTPFsConfig) isUnixDomainSocket() bool {
  55. return strings.HasPrefix(c.Endpoint, "http://unix") || strings.HasPrefix(c.Endpoint, "https://unix")
  56. }
  57. // HideConfidentialData hides confidential data
  58. func (c *HTTPFsConfig) HideConfidentialData() {
  59. if c.Password != nil {
  60. c.Password.Hide()
  61. }
  62. if c.APIKey != nil {
  63. c.APIKey.Hide()
  64. }
  65. }
  66. func (c *HTTPFsConfig) setNilSecretsIfEmpty() {
  67. if c.Password != nil && c.Password.IsEmpty() {
  68. c.Password = nil
  69. }
  70. if c.APIKey != nil && c.APIKey.IsEmpty() {
  71. c.APIKey = nil
  72. }
  73. }
  74. func (c *HTTPFsConfig) setEmptyCredentialsIfNil() {
  75. if c.Password == nil {
  76. c.Password = kms.NewEmptySecret()
  77. }
  78. if c.APIKey == nil {
  79. c.APIKey = kms.NewEmptySecret()
  80. }
  81. }
  82. func (c *HTTPFsConfig) isEqual(other HTTPFsConfig) bool {
  83. if c.Endpoint != other.Endpoint {
  84. return false
  85. }
  86. if c.Username != other.Username {
  87. return false
  88. }
  89. if c.SkipTLSVerify != other.SkipTLSVerify {
  90. return false
  91. }
  92. c.setEmptyCredentialsIfNil()
  93. other.setEmptyCredentialsIfNil()
  94. if !c.Password.IsEqual(other.Password) {
  95. return false
  96. }
  97. return c.APIKey.IsEqual(other.APIKey)
  98. }
  99. func (c *HTTPFsConfig) isSameResource(other HTTPFsConfig) bool {
  100. if c.EqualityCheckMode > 0 || other.EqualityCheckMode > 0 {
  101. if c.Username != other.Username {
  102. return false
  103. }
  104. }
  105. return c.Endpoint == other.Endpoint
  106. }
  107. // validate returns an error if the configuration is not valid
  108. func (c *HTTPFsConfig) validate() error {
  109. c.setEmptyCredentialsIfNil()
  110. if c.Endpoint == "" {
  111. return errors.New("httpfs: endpoint cannot be empty")
  112. }
  113. c.Endpoint = strings.TrimRight(c.Endpoint, "/")
  114. endpointURL, err := url.Parse(c.Endpoint)
  115. if err != nil {
  116. return fmt.Errorf("httpfs: invalid endpoint: %w", err)
  117. }
  118. if !util.IsStringPrefixInSlice(c.Endpoint, supportedEndpointSchema) {
  119. return errors.New("httpfs: invalid endpoint schema: http and https are supported")
  120. }
  121. if endpointURL.Host == "unix" {
  122. socketPath := endpointURL.Query().Get("socket_path")
  123. if !filepath.IsAbs(socketPath) {
  124. return fmt.Errorf("httpfs: invalid unix domain socket path: %q", socketPath)
  125. }
  126. }
  127. if !isEqualityCheckModeValid(c.EqualityCheckMode) {
  128. return errors.New("invalid equality_check_mode")
  129. }
  130. if c.Password.IsEncrypted() && !c.Password.IsValid() {
  131. return errors.New("httpfs: invalid encrypted password")
  132. }
  133. if !c.Password.IsEmpty() && !c.Password.IsValidInput() {
  134. return errors.New("httpfs: invalid password")
  135. }
  136. if c.APIKey.IsEncrypted() && !c.APIKey.IsValid() {
  137. return errors.New("httpfs: invalid encrypted API key")
  138. }
  139. if !c.APIKey.IsEmpty() && !c.APIKey.IsValidInput() {
  140. return errors.New("httpfs: invalid API key")
  141. }
  142. return nil
  143. }
  144. // ValidateAndEncryptCredentials validates the config and encrypts credentials if they are in plain text
  145. func (c *HTTPFsConfig) ValidateAndEncryptCredentials(additionalData string) error {
  146. if err := c.validate(); err != nil {
  147. return util.NewValidationError(fmt.Sprintf("could not validate HTTP fs config: %v", err))
  148. }
  149. if c.Password.IsPlain() {
  150. c.Password.SetAdditionalData(additionalData)
  151. if err := c.Password.Encrypt(); err != nil {
  152. return util.NewValidationError(fmt.Sprintf("could not encrypt HTTP fs password: %v", err))
  153. }
  154. }
  155. if c.APIKey.IsPlain() {
  156. c.APIKey.SetAdditionalData(additionalData)
  157. if err := c.APIKey.Encrypt(); err != nil {
  158. return util.NewValidationError(fmt.Sprintf("could not encrypt HTTP fs API key: %v", err))
  159. }
  160. }
  161. return nil
  162. }
  163. // HTTPFs is a Fs implementation for the SFTPGo HTTP filesystem backend
  164. type HTTPFs struct {
  165. connectionID string
  166. localTempDir string
  167. // if not empty this fs is mouted as virtual folder in the specified path
  168. mountPath string
  169. config *HTTPFsConfig
  170. client *http.Client
  171. ctxTimeout time.Duration
  172. }
  173. // NewHTTPFs returns an HTTPFs object that allows to interact with SFTPGo HTTP filesystem backends
  174. func NewHTTPFs(connectionID, localTempDir, mountPath string, config HTTPFsConfig) (Fs, error) {
  175. if localTempDir == "" {
  176. localTempDir = getLocalTempDir()
  177. }
  178. config.setEmptyCredentialsIfNil()
  179. if !config.Password.IsEmpty() {
  180. if err := config.Password.TryDecrypt(); err != nil {
  181. return nil, err
  182. }
  183. }
  184. if !config.APIKey.IsEmpty() {
  185. if err := config.APIKey.TryDecrypt(); err != nil {
  186. return nil, err
  187. }
  188. }
  189. fs := &HTTPFs{
  190. connectionID: connectionID,
  191. localTempDir: localTempDir,
  192. mountPath: mountPath,
  193. config: &config,
  194. ctxTimeout: 30 * time.Second,
  195. }
  196. transport := http.DefaultTransport.(*http.Transport).Clone()
  197. transport.MaxResponseHeaderBytes = 1 << 16
  198. transport.WriteBufferSize = 1 << 16
  199. transport.ReadBufferSize = 1 << 16
  200. if fs.config.isUnixDomainSocket() {
  201. endpointURL, err := url.Parse(fs.config.Endpoint)
  202. if err != nil {
  203. return nil, err
  204. }
  205. if endpointURL.Host == "unix" {
  206. socketPath := endpointURL.Query().Get("socket_path")
  207. if !filepath.IsAbs(socketPath) {
  208. return nil, fmt.Errorf("httpfs: invalid unix domain socket path: %q", socketPath)
  209. }
  210. if endpointURL.Scheme == "https" {
  211. transport.DialTLSContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
  212. var tlsConfig *tls.Config
  213. var d tls.Dialer
  214. if config.SkipTLSVerify {
  215. tlsConfig = getInsecureTLSConfig()
  216. }
  217. d.Config = tlsConfig
  218. return d.DialContext(ctx, "unix", socketPath)
  219. }
  220. } else {
  221. transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
  222. var d net.Dialer
  223. return d.DialContext(ctx, "unix", socketPath)
  224. }
  225. }
  226. endpointURL.Path = path.Join(endpointURL.Path, endpointURL.Query().Get("api_prefix"))
  227. endpointURL.RawQuery = ""
  228. endpointURL.RawFragment = ""
  229. fs.config.Endpoint = endpointURL.String()
  230. }
  231. }
  232. if config.SkipTLSVerify {
  233. if transport.TLSClientConfig != nil {
  234. transport.TLSClientConfig.InsecureSkipVerify = true
  235. } else {
  236. transport.TLSClientConfig = getInsecureTLSConfig()
  237. }
  238. }
  239. fs.client = &http.Client{
  240. Transport: transport,
  241. }
  242. return fs, nil
  243. }
  244. // Name returns the name for the Fs implementation
  245. func (fs *HTTPFs) Name() string {
  246. return fmt.Sprintf("%v %q", httpFsName, fs.config.Endpoint)
  247. }
  248. // ConnectionID returns the connection ID associated to this Fs implementation
  249. func (fs *HTTPFs) ConnectionID() string {
  250. return fs.connectionID
  251. }
  252. // Stat returns a FileInfo describing the named file
  253. func (fs *HTTPFs) Stat(name string) (os.FileInfo, error) {
  254. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  255. defer cancelFn()
  256. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "stat", name, "", "", nil)
  257. if err != nil {
  258. return nil, err
  259. }
  260. defer resp.Body.Close()
  261. respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPFsResponseSize))
  262. if err != nil {
  263. return nil, err
  264. }
  265. var response statResponse
  266. err = json.Unmarshal(respBody, &response)
  267. if err != nil {
  268. return nil, err
  269. }
  270. return response.getFileInfo(), nil
  271. }
  272. // Lstat returns a FileInfo describing the named file
  273. func (fs *HTTPFs) Lstat(name string) (os.FileInfo, error) {
  274. return fs.Stat(name)
  275. }
  276. // Open opens the named file for reading
  277. func (fs *HTTPFs) Open(name string, offset int64) (File, *PipeReader, func(), error) {
  278. r, w, err := pipeat.PipeInDir(fs.localTempDir)
  279. if err != nil {
  280. return nil, nil, nil, err
  281. }
  282. p := NewPipeReader(r)
  283. ctx, cancelFn := context.WithCancel(context.Background())
  284. var queryString string
  285. if offset > 0 {
  286. queryString = fmt.Sprintf("?offset=%d", offset)
  287. }
  288. go func() {
  289. defer cancelFn()
  290. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "open", name, queryString, "", nil)
  291. if err != nil {
  292. fsLog(fs, logger.LevelError, "download error, path %q, err: %v", name, err)
  293. w.CloseWithError(err) //nolint:errcheck
  294. metric.HTTPFsTransferCompleted(0, 1, err)
  295. return
  296. }
  297. defer resp.Body.Close()
  298. n, err := io.Copy(w, resp.Body)
  299. w.CloseWithError(err) //nolint:errcheck
  300. fsLog(fs, logger.LevelDebug, "download completed, path %q size: %v, err: %+v", name, n, err)
  301. metric.HTTPFsTransferCompleted(n, 1, err)
  302. }()
  303. return nil, p, cancelFn, nil
  304. }
  305. // Create creates or opens the named file for writing
  306. func (fs *HTTPFs) Create(name string, flag, checks int) (File, PipeWriter, func(), error) {
  307. r, w, err := pipeat.PipeInDir(fs.localTempDir)
  308. if err != nil {
  309. return nil, nil, nil, err
  310. }
  311. p := NewPipeWriter(w)
  312. ctx, cancelFn := context.WithCancel(context.Background())
  313. go func() {
  314. defer cancelFn()
  315. contentType := mime.TypeByExtension(path.Ext(name))
  316. queryString := fmt.Sprintf("?flags=%d&checks=%d", flag, checks)
  317. resp, err := fs.sendHTTPRequest(ctx, http.MethodPost, "create", name, queryString, contentType,
  318. &wrapReader{reader: r})
  319. if err != nil {
  320. fsLog(fs, logger.LevelError, "upload error, path %q, err: %v", name, err)
  321. r.CloseWithError(err) //nolint:errcheck
  322. p.Done(err)
  323. metric.HTTPFsTransferCompleted(0, 0, err)
  324. return
  325. }
  326. defer resp.Body.Close()
  327. r.CloseWithError(err) //nolint:errcheck
  328. p.Done(err)
  329. fsLog(fs, logger.LevelDebug, "upload completed, path: %q, readed bytes: %d", name, r.GetReadedBytes())
  330. metric.HTTPFsTransferCompleted(r.GetReadedBytes(), 0, err)
  331. }()
  332. return nil, p, cancelFn, nil
  333. }
  334. // Rename renames (moves) source to target.
  335. func (fs *HTTPFs) Rename(source, target string) (int, int64, error) {
  336. if source == target {
  337. return -1, -1, nil
  338. }
  339. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  340. defer cancelFn()
  341. queryString := fmt.Sprintf("?target=%s", url.QueryEscape(target))
  342. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "rename", source, queryString, "", nil)
  343. if err != nil {
  344. return -1, -1, err
  345. }
  346. defer resp.Body.Close()
  347. return -1, -1, nil
  348. }
  349. // Remove removes the named file or (empty) directory.
  350. func (fs *HTTPFs) Remove(name string, _ bool) error {
  351. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  352. defer cancelFn()
  353. resp, err := fs.sendHTTPRequest(ctx, http.MethodDelete, "remove", name, "", "", nil)
  354. if err != nil {
  355. return err
  356. }
  357. defer resp.Body.Close()
  358. return nil
  359. }
  360. // Mkdir creates a new directory with the specified name and default permissions
  361. func (fs *HTTPFs) Mkdir(name string) error {
  362. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  363. defer cancelFn()
  364. resp, err := fs.sendHTTPRequest(ctx, http.MethodPost, "mkdir", name, "", "", nil)
  365. if err != nil {
  366. return err
  367. }
  368. defer resp.Body.Close()
  369. return nil
  370. }
  371. // Symlink creates source as a symbolic link to target.
  372. func (*HTTPFs) Symlink(_, _ string) error {
  373. return ErrVfsUnsupported
  374. }
  375. // Readlink returns the destination of the named symbolic link
  376. func (*HTTPFs) Readlink(_ string) (string, error) {
  377. return "", ErrVfsUnsupported
  378. }
  379. // Chown changes the numeric uid and gid of the named file.
  380. func (fs *HTTPFs) Chown(_ string, _ int, _ int) error {
  381. return ErrVfsUnsupported
  382. }
  383. // Chmod changes the mode of the named file to mode.
  384. func (fs *HTTPFs) Chmod(name string, mode os.FileMode) error {
  385. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  386. defer cancelFn()
  387. queryString := fmt.Sprintf("?mode=%d", mode)
  388. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "chmod", name, queryString, "", nil)
  389. if err != nil {
  390. return err
  391. }
  392. defer resp.Body.Close()
  393. return nil
  394. }
  395. // Chtimes changes the access and modification times of the named file.
  396. func (fs *HTTPFs) Chtimes(name string, atime, mtime time.Time, _ bool) error {
  397. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  398. defer cancelFn()
  399. queryString := fmt.Sprintf("?access_time=%s&modification_time=%s", atime.UTC().Format(time.RFC3339),
  400. mtime.UTC().Format(time.RFC3339))
  401. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "chtimes", name, queryString, "", nil)
  402. if err != nil {
  403. return err
  404. }
  405. defer resp.Body.Close()
  406. return nil
  407. }
  408. // Truncate changes the size of the named file.
  409. // Truncate by path is not supported, while truncating an opened
  410. // file is handled inside base transfer
  411. func (fs *HTTPFs) Truncate(name string, size int64) error {
  412. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  413. defer cancelFn()
  414. queryString := fmt.Sprintf("?size=%d", size)
  415. resp, err := fs.sendHTTPRequest(ctx, http.MethodPatch, "truncate", name, queryString, "", nil)
  416. if err != nil {
  417. return err
  418. }
  419. defer resp.Body.Close()
  420. return nil
  421. }
  422. // ReadDir reads the directory named by dirname and returns
  423. // a list of directory entries.
  424. func (fs *HTTPFs) ReadDir(dirname string) ([]os.FileInfo, error) {
  425. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  426. defer cancelFn()
  427. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "readdir", dirname, "", "", nil)
  428. if err != nil {
  429. return nil, err
  430. }
  431. defer resp.Body.Close()
  432. respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPFsResponseSize*10))
  433. if err != nil {
  434. return nil, err
  435. }
  436. var response []statResponse
  437. err = json.Unmarshal(respBody, &response)
  438. if err != nil {
  439. return nil, err
  440. }
  441. result := make([]os.FileInfo, 0, len(response))
  442. for _, stat := range response {
  443. result = append(result, stat.getFileInfo())
  444. }
  445. return result, nil
  446. }
  447. // IsUploadResumeSupported returns true if resuming uploads is supported.
  448. func (*HTTPFs) IsUploadResumeSupported() bool {
  449. return false
  450. }
  451. // IsConditionalUploadResumeSupported returns if resuming uploads is supported
  452. // for the specified size
  453. func (*HTTPFs) IsConditionalUploadResumeSupported(_ int64) bool {
  454. return false
  455. }
  456. // IsAtomicUploadSupported returns true if atomic upload is supported.
  457. func (*HTTPFs) IsAtomicUploadSupported() bool {
  458. return false
  459. }
  460. // IsNotExist returns a boolean indicating whether the error is known to
  461. // report that a file or directory does not exist
  462. func (*HTTPFs) IsNotExist(err error) bool {
  463. return errors.Is(err, fs.ErrNotExist)
  464. }
  465. // IsPermission returns a boolean indicating whether the error is known to
  466. // report that permission is denied.
  467. func (*HTTPFs) IsPermission(err error) bool {
  468. return errors.Is(err, fs.ErrPermission)
  469. }
  470. // IsNotSupported returns true if the error indicate an unsupported operation
  471. func (*HTTPFs) IsNotSupported(err error) bool {
  472. if err == nil {
  473. return false
  474. }
  475. return err == ErrVfsUnsupported
  476. }
  477. // CheckRootPath creates the specified local root directory if it does not exists
  478. func (fs *HTTPFs) CheckRootPath(username string, uid int, gid int) bool {
  479. // we need a local directory for temporary files
  480. osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, "", nil)
  481. return osFs.CheckRootPath(username, uid, gid)
  482. }
  483. // ScanRootDirContents returns the number of files and their size
  484. func (fs *HTTPFs) ScanRootDirContents() (int, int64, error) {
  485. return fs.GetDirSize("/")
  486. }
  487. // CheckMetadata checks the metadata consistency
  488. func (*HTTPFs) CheckMetadata() error {
  489. return nil
  490. }
  491. // GetDirSize returns the number of files and the size for a folder
  492. // including any subfolders
  493. func (fs *HTTPFs) GetDirSize(dirname string) (int, int64, error) {
  494. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  495. defer cancelFn()
  496. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "dirsize", dirname, "", "", nil)
  497. if err != nil {
  498. return 0, 0, err
  499. }
  500. defer resp.Body.Close()
  501. respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPFsResponseSize))
  502. if err != nil {
  503. return 0, 0, err
  504. }
  505. var response dirSizeResponse
  506. err = json.Unmarshal(respBody, &response)
  507. if err != nil {
  508. return 0, 0, err
  509. }
  510. return response.Files, response.Size, nil
  511. }
  512. // GetAtomicUploadPath returns the path to use for an atomic upload.
  513. func (*HTTPFs) GetAtomicUploadPath(_ string) string {
  514. return ""
  515. }
  516. // GetRelativePath returns the path for a file relative to the user's home dir.
  517. // This is the path as seen by SFTPGo users
  518. func (fs *HTTPFs) GetRelativePath(name string) string {
  519. rel := path.Clean(name)
  520. if rel == "." {
  521. rel = ""
  522. }
  523. if !path.IsAbs(rel) {
  524. rel = "/" + rel
  525. }
  526. if fs.mountPath != "" {
  527. rel = path.Join(fs.mountPath, rel)
  528. }
  529. return rel
  530. }
  531. // Walk walks the file tree rooted at root, calling walkFn for each file or
  532. // directory in the tree, including root. The result are unordered
  533. func (fs *HTTPFs) Walk(root string, walkFn filepath.WalkFunc) error {
  534. info, err := fs.Lstat(root)
  535. if err != nil {
  536. return walkFn(root, nil, err)
  537. }
  538. return fs.walk(root, info, walkFn)
  539. }
  540. // Join joins any number of path elements into a single path
  541. func (*HTTPFs) Join(elem ...string) string {
  542. return strings.TrimPrefix(path.Join(elem...), "/")
  543. }
  544. // HasVirtualFolders returns true if folders are emulated
  545. func (*HTTPFs) HasVirtualFolders() bool {
  546. return false
  547. }
  548. // ResolvePath returns the matching filesystem path for the specified virtual path
  549. func (fs *HTTPFs) ResolvePath(virtualPath string) (string, error) {
  550. if fs.mountPath != "" {
  551. virtualPath = strings.TrimPrefix(virtualPath, fs.mountPath)
  552. }
  553. if !path.IsAbs(virtualPath) {
  554. virtualPath = path.Clean("/" + virtualPath)
  555. }
  556. return virtualPath, nil
  557. }
  558. // GetMimeType returns the content type
  559. func (fs *HTTPFs) GetMimeType(name string) (string, error) {
  560. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  561. defer cancelFn()
  562. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "stat", name, "", "", nil)
  563. if err != nil {
  564. return "", err
  565. }
  566. defer resp.Body.Close()
  567. respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPFsResponseSize))
  568. if err != nil {
  569. return "", err
  570. }
  571. var response mimeTypeResponse
  572. err = json.Unmarshal(respBody, &response)
  573. if err != nil {
  574. return "", err
  575. }
  576. return response.Mime, nil
  577. }
  578. // Close closes the fs
  579. func (fs *HTTPFs) Close() error {
  580. fs.client.CloseIdleConnections()
  581. return nil
  582. }
  583. // GetAvailableDiskSize returns the available size for the specified path
  584. func (fs *HTTPFs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) {
  585. ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
  586. defer cancelFn()
  587. resp, err := fs.sendHTTPRequest(ctx, http.MethodGet, "statvfs", dirName, "", "", nil)
  588. if err != nil {
  589. return nil, err
  590. }
  591. defer resp.Body.Close()
  592. respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPFsResponseSize))
  593. if err != nil {
  594. return nil, err
  595. }
  596. var response statVFSResponse
  597. err = json.Unmarshal(respBody, &response)
  598. if err != nil {
  599. return nil, err
  600. }
  601. return response.toSFTPStatVFS(), nil
  602. }
  603. func (fs *HTTPFs) sendHTTPRequest(ctx context.Context, method, base, name, queryString, contentType string,
  604. body io.Reader,
  605. ) (*http.Response, error) {
  606. url := fmt.Sprintf("%s/%s/%s%s", fs.config.Endpoint, base, url.PathEscape(name), queryString)
  607. req, err := http.NewRequest(method, url, body)
  608. if err != nil {
  609. return nil, err
  610. }
  611. if contentType != "" {
  612. req.Header.Set("Content-Type", contentType)
  613. }
  614. if fs.config.APIKey.GetPayload() != "" {
  615. req.Header.Set("X-API-KEY", fs.config.APIKey.GetPayload())
  616. }
  617. if fs.config.Username != "" || fs.config.Password.GetPayload() != "" {
  618. req.SetBasicAuth(fs.config.Username, fs.config.Password.GetPayload())
  619. }
  620. resp, err := fs.client.Do(req.WithContext(ctx))
  621. if err != nil {
  622. return nil, fmt.Errorf("unable to send HTTP request to URL %v: %w", url, err)
  623. }
  624. if err = getErrorFromResponseCode(resp.StatusCode); err != nil {
  625. resp.Body.Close()
  626. return nil, err
  627. }
  628. return resp, nil
  629. }
  630. // walk recursively descends path, calling walkFn.
  631. func (fs *HTTPFs) walk(filePath string, info fs.FileInfo, walkFn filepath.WalkFunc) error {
  632. if !info.IsDir() {
  633. return walkFn(filePath, info, nil)
  634. }
  635. files, err := fs.ReadDir(filePath)
  636. err1 := walkFn(filePath, info, err)
  637. if err != nil || err1 != nil {
  638. return err1
  639. }
  640. for _, fi := range files {
  641. objName := path.Join(filePath, fi.Name())
  642. err = fs.walk(objName, fi, walkFn)
  643. if err != nil {
  644. return err
  645. }
  646. }
  647. return nil
  648. }
  649. func getErrorFromResponseCode(code int) error {
  650. switch code {
  651. case 401, 403:
  652. return os.ErrPermission
  653. case 404:
  654. return os.ErrNotExist
  655. case 501:
  656. return ErrVfsUnsupported
  657. case 200, 201:
  658. return nil
  659. default:
  660. return fmt.Errorf("unexpected response code: %v", code)
  661. }
  662. }
  663. func getInsecureTLSConfig() *tls.Config {
  664. return &tls.Config{
  665. InsecureSkipVerify: true,
  666. }
  667. }
  668. type wrapReader struct {
  669. reader io.Reader
  670. }
  671. func (r *wrapReader) Read(p []byte) (n int, err error) {
  672. return r.reader.Read(p)
  673. }
  674. type statResponse struct {
  675. Name string `json:"name"`
  676. Size int64 `json:"size"`
  677. Mode uint32 `json:"mode"`
  678. LastModified time.Time `json:"last_modified"`
  679. }
  680. func (s *statResponse) getFileInfo() os.FileInfo {
  681. info := NewFileInfo(s.Name, false, s.Size, s.LastModified, false)
  682. info.SetMode(fs.FileMode(s.Mode))
  683. return info
  684. }
  685. type dirSizeResponse struct {
  686. Files int `json:"files"`
  687. Size int64 `json:"size"`
  688. }
  689. type mimeTypeResponse struct {
  690. Mime string `json:"mime"`
  691. }
  692. type statVFSResponse struct {
  693. ID uint32 `json:"-"`
  694. Bsize uint64 `json:"bsize"`
  695. Frsize uint64 `json:"frsize"`
  696. Blocks uint64 `json:"blocks"`
  697. Bfree uint64 `json:"bfree"`
  698. Bavail uint64 `json:"bavail"`
  699. Files uint64 `json:"files"`
  700. Ffree uint64 `json:"ffree"`
  701. Favail uint64 `json:"favail"`
  702. Fsid uint64 `json:"fsid"`
  703. Flag uint64 `json:"flag"`
  704. Namemax uint64 `json:"namemax"`
  705. }
  706. func (s *statVFSResponse) toSFTPStatVFS() *sftp.StatVFS {
  707. return &sftp.StatVFS{
  708. Bsize: s.Bsize,
  709. Frsize: s.Frsize,
  710. Blocks: s.Blocks,
  711. Bfree: s.Bfree,
  712. Bavail: s.Bavail,
  713. Files: s.Files,
  714. Ffree: s.Ffree,
  715. Favail: s.Ffree,
  716. Flag: s.Flag,
  717. Namemax: s.Namemax,
  718. }
  719. }