Issue 2325 – Gracefully handle invalid Unicode characters FFFF and FFFE in filenames in .NET assembly

https://winscp.net/tracker/2325

As a side effect this also "fixes" unexpected encoding of invalid character replacement in session log file (value 1 was encoded as 0100, instead of expected 0001)

Source commit: fb80501de79abd3bbd2d4d994252029f5f7bda88

source/core/Common.cpp

@@ -1422,7 +1422,8 @@ UnicodeString __fastcall BytesToHex(RawByteString Str, bool UpperCase, wchar_t S
 //---------------------------------------------------------------------------
 UnicodeString __fastcall CharToHex(wchar_t Ch, bool UpperCase)
 {
-  return BytesToHex(reinterpret_cast<const unsigned char *>(&Ch), sizeof(Ch), UpperCase);
+  // BytesToHex would encode with opposite/unexpected endianness
+  return ByteToHex(Ch >> 8, UpperCase) + ByteToHex(Ch & 0xFF, UpperCase);
 }
 //---------------------------------------------------------------------------
 RawByteString __fastcall HexToBytes(const UnicodeString Hex)
@@ -2889,6 +2890,7 @@ UnicodeString __fastcall DoEncodeUrl(UnicodeString S, const UnicodeString & DoNo
     {
       UTF8String UtfS(S.SubString(Index, 1));
       UnicodeString H;
+      // BytesToHex with separator would do the same
       for (int Index2 = 1; Index2 <= UtfS.Length(); Index2++)
       {
         H += L"%" + ByteToHex(static_cast<unsigned char>(UtfS[Index2]));

source/core/SessionInfo.cpp

@@ -24,7 +24,8 @@ UnicodeString __fastcall DoXmlEscape(UnicodeString Str, bool NewLine)
   for (int i = 1; i <= Str.Length(); i++)
   {
     UnicodeString Repl;
-    switch (Str[i])
+    wchar_t Ch = Str[i];
+    switch (Ch)
     {
       case L'\x00': // \0 Is not valid in XML anyway
       case L'\x01':
@@ -57,7 +58,12 @@ UnicodeString __fastcall DoXmlEscape(UnicodeString Str, bool NewLine)
       case L'\x1D':
       case L'\x1E':
       case L'\x1F':
-        Repl = L"#x" + ByteToHex((unsigned char)Str[i]) + L";";
+        Repl = L"#x" + ByteToHex((unsigned char)Ch) + L";";
+        break;
+
+      case L'\xFFFE':
+      case L'\xFFFF':
+        Repl = L"#x" + CharToHex(Ch) + L";";
         break;
 
       case L'&':