Bug 1824: Workaround for specific encoding of special characters in filenames by OneDrive WebDAV interface

https://winscp.net/tracker/1824
(cherry picked from commit 9a5f22bd87f01df41cdd60c929f95aa1df062c2f)

Source commit: 52f42c279c9ecc8e9b550d4d359c56eb6dd0b37c

libs/neon/src/ne_207.c

@@ -188,16 +188,7 @@ end_element(void *userdata, int state, const char *nspace, const char *name)
                 cdata = hh;
             }
 
-            #ifdef WINSCP
-            if (p->flags & NE_207_LIBERAL_ESCAPING) {
-                hh = ne_path_escapef(cdata, NE_PATH_NONPC);
-                NE_DEBUG(NE_DBG_XML, "207: Enabled liberal href escaping [%s]->[%s]\n",
-                         cdata, hh);
-                cdata = hh;
-            }
-            #endif
-
-            ret = ne_uri_parse(cdata, &ref);
+            ret = ne_uri_parse_ex(cdata, &ref, (p->flags & NE_207_LIBERAL_ESCAPING)); // WINSCP
             if (!ret) {
                 ne_uri_resolve(&p->base, &ref, &resolved);
 

libs/neon/src/ne_uri.c

@@ -152,7 +152,7 @@ unsigned int ne_uri_defaultport(const char *scheme)
 	return 0;
 }
 
-int ne_uri_parse(const char *uri, ne_uri *parsed)
+int ne_uri_parse_ex(const char *uri, ne_uri *parsed, int liberal) // WINSCP
 {
     const char *p, *s;
 
@@ -238,7 +238,11 @@ int ne_uri_parse(const char *uri, ne_uri *parsed)
 
     p = s;
 
-    while (uri_lookup(*p) & URI_SEGCHAR)
+    while ((uri_lookup(*p) & URI_SEGCHAR) ||
+           (liberal && // WINSCP
+            ((uri_lookup(*p) & SD) ||
+            ((uri_lookup(*p) & OT) && (*p) >= ' ') || // OT without control characters
+            ((*p) == '[') || ((*p) == ']')))) // GD without #
         p++;
 
     /* => p = [ "?" query ] [ "#" fragment ] */
@@ -279,6 +283,13 @@ int ne_uri_parse(const char *uri, ne_uri *parsed)
     return 0;
 }
 
+#ifdef WINSCP
+int ne_uri_parse(const char *uri, ne_uri *parsed)
+{
+    return ne_uri_parse_ex(uri, parsed, 0);
+}
+#endif
+
 /* This function directly implements the "Merge Paths" algorithm
  * described in RFC 3986 section 5.2.3. */
 static char *merge_paths(const ne_uri *base, const char *path)

libs/neon/src/ne_uri.h

@@ -84,7 +84,8 @@ typedef struct {
  * NULL, or point to malloc-allocated NUL-terminated strings;
  * ne_uri_free can be used to free any set fields.  On success,
  * parsed->path is guaranteed to be non-NULL. */
-int ne_uri_parse(const char *uri, ne_uri *parsed);
+int ne_uri_parse(const char *uri, ne_uri *parsed); // WINSCP
+int ne_uri_parse_ex(const char *uri, ne_uri *parsed, int liberal); // WINSCP
 
 /* Turns a URI structure back into a string.  The returned string is
  * malloc-allocated, and must be freed by the caller. */

source/core/WebDAVFileSystem.cpp

@@ -65,11 +65,22 @@ static AnsiString PathEscape(const char * Path)
   return Result;
 }
 //---------------------------------------------------------------------------
-static UTF8String PathUnescape(const char * Path)
+static UnicodeString PathUnescape(const char * Path)
 {
   char * UnescapedPath = ne_path_unescape(Path);
-  UTF8String Result = UnescapedPath;
-  ne_free(UnescapedPath);
+  UTF8String UtfResult;
+  if (UnescapedPath != NULL)
+  {
+    UtfResult = UnescapedPath;
+    ne_free(UnescapedPath);
+  }
+  else
+  {
+    // OneDrive, particularly in the response to *file* PROPFIND tend to return a malformed URL.
+    // In such case, take the path as is and we will probably overwrite the name with "display name".
+    UtfResult = Path;
+  }
+  UnicodeString Result = StrFromNeon(UtfResult);
   return Result;
 }
 //---------------------------------------------------------------------------
@@ -193,6 +204,13 @@ void __fastcall TWebDAVFileSystem::Open()
   FSessionInfo.CertificateVerifiedManually = false;
 
   UnicodeString HostName = Data->HostNameExpanded;
+
+  FOneDrive = SameText(HostName, L"d.docs.live.net");
+  if (FOneDrive)
+  {
+    FTerminal->LogEvent(L"OneDrive host detected.");
+  }
+
   size_t Port = Data->PortNumber;
   UnicodeString ProtocolName = (FTerminal->SessionData->Ftps == ftpsNone) ? HttpProtocol : HttpsProtocol;
   UnicodeString Path = Data->RemoteDirectory;
@@ -221,7 +239,7 @@ UnicodeString __fastcall TWebDAVFileSystem::ParsePathFromUrl(const UnicodeString
   ne_uri ParsedUri;
   if (ne_uri_parse(StrToNeon(Url), &ParsedUri) == 0)
   {
-    Result = StrFromNeon(PathUnescape(ParsedUri.path));
+    Result = PathUnescape(ParsedUri.path);
     ne_uri_free(&ParsedUri);
   }
   return Result;
@@ -294,7 +312,8 @@ void __fastcall TWebDAVFileSystem::InitSession(ne_session_s * Session)
 
   ne_set_session_private(Session, SESSION_FS_KEY, this);
 
-  ne_set_session_flag(Session, NE_SESSFLAG_LIBERAL_ESCAPING, Data->WebDavLiberalEscaping);
+  // Allow ^-escaping in OneDrive
+  ne_set_session_flag(Session, NE_SESSFLAG_LIBERAL_ESCAPING, Data->WebDavLiberalEscaping || FOneDrive);
 }
 //---------------------------------------------------------------------------
 void TWebDAVFileSystem::NeonOpen(UnicodeString & CorrectedUrl, const UnicodeString & Url)
@@ -803,7 +822,7 @@ void __fastcall TWebDAVFileSystem::ReadFile(const UnicodeString FileName,
 void TWebDAVFileSystem::NeonPropsResult(
   void * UserData, const ne_uri * Uri, const ne_prop_result_set * Results)
 {
-  UnicodeString Path = StrFromNeon(PathUnescape(Uri->path).c_str());
+  UnicodeString Path = PathUnescape(Uri->path);
 
   TReadFileData & Data = *static_cast<TReadFileData *>(UserData);
   if (Data.FileList != NULL)
@@ -838,7 +857,7 @@ void __fastcall TWebDAVFileSystem::ParsePropResultSet(TRemoteFile * File,
 {
   File->FullFileName = UnixExcludeTrailingBackslash(Path);
   // Some servers do not use DAV:collection tag, but indicate the folder by trailing slash only.
-  // It seems that all servers actually use the trailing slash, including IIS, mod_Dav, IT Hit, OpenDrive, etc.
+  // But not all, for example OneDrive does not (it did in the past).
   bool Collection = (File->FullFileName != Path);
   File->FileName = UnixExtractFileName(File->FullFileName);
   const char * ContentLength = GetProp(Results, PROP_CONTENT_LENGTH);
@@ -888,8 +907,7 @@ void __fastcall TWebDAVFileSystem::ParsePropResultSet(TRemoteFile * File,
   // optimization
   if (!Collection)
   {
-    // This is possibly redundant code as all servers we know (see a comment above)
-    // indicate the folder by trailing slash too
+    // See a comment at the Collection declaration.
     const char * ResourceType = GetProp(Results, PROP_RESOURCE_TYPE);
     if (ResourceType != NULL)
     {
@@ -921,6 +939,17 @@ void __fastcall TWebDAVFileSystem::ParsePropResultSet(TRemoteFile * File,
   if (DisplayName != NULL)
   {
     File->DisplayName = StrFromNeon(DisplayName);
+    // OneDrive caret escaping (we could do this for all files, but let's limit the scope for now).
+    // In *file* PROPFIND response, the # (and other symbols like comma or plus) is not escaped at all
+    // (while in directory listing, they are caret-escaped),
+    // so if we see one in the display name, take the name from there.
+    // * and % won't help, as OneDrive seem to have bug with % as the end of the filename,
+    // and the * (and others) is removed from file names.
+    if (FOneDrive && (ContainsText(File->FileName, L"^") || (wcspbrk(File->DisplayName.c_str(), L"&,+#[]%*") != NULL)))
+    {
+      File->FileName = File->DisplayName;
+      File->FullFileName = UnixCombinePaths(UnixExtractFileDir(File->FullFileName), File->FileName);
+    }
   }
 
   const UnicodeString RightsDelimiter(L", ");

source/core/WebDAVFileSystem.h

@@ -168,6 +168,7 @@ private:
   UnicodeString FLastAuthorizationProtocol;
   bool FAuthenticationRetry;
   bool FNtlmAuthenticationFailed;
+  bool FOneDrive;
 
   void __fastcall CustomReadFile(UnicodeString FileName,
     TRemoteFile *& File, TRemoteFile * ALinkedByFile);