With SFTP protocol files can be streamed to stdout in scripting using chunked encoding to distinguish multiple files (part of Bug 1738)

(cherry picked from commit c86def11d847963e846ea33fa864e21aba0a0c87)

Source commit: 535a8d33c450b5d74786de1fc8b480c5a663f191

dotnet/internal/ChunkedReadStream.cs

@@ -0,0 +1,113 @@
+using System;
+using System.Globalization;
+using System.IO;
+
+namespace WinSCP
+{
+    internal class ChunkedReadStream : Stream
+    {
+        public ChunkedReadStream(Stream baseStream)
+        {
+            _baseStream = baseStream;
+            _remaining = 0;
+            _eof = false;
+        }
+
+        public override bool CanRead => !_eof;
+
+        public override bool CanSeek => false;
+
+        public override bool CanWrite => false;
+
+        public override long Length => throw new NotImplementedException();
+
+        public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
+
+        public override void Flush()
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            int result;
+            if (_eof)
+            {
+                result = 0;
+            }
+            else
+            {
+                if (_remaining == 0)
+                {
+                    string lenStr = string.Empty;
+                    while (!lenStr.EndsWith("\r\n"))
+                    {
+                        if (lenStr.Length > 64)
+                        {
+                            throw new Exception("Too long chunk length line");
+                        }
+                        int b = _baseStream.ReadByte();
+                        if (b < 0)
+                        {
+                            throw new Exception("End of stream reached while reading chunk length line");
+                        }
+                        lenStr += (char)b;
+                    }
+                    lenStr = lenStr.Trim();
+                    _remaining = int.Parse(lenStr, NumberStyles.HexNumber);
+                    if (_remaining == 0)
+                    {
+                        _eof = true;
+                    }
+                }
+
+                // Not sure if it is ok to call Read with 0
+                if (_remaining > 0)
+                {
+                    int read = Math.Min(count, _remaining);
+                    result = _baseStream.Read(buffer, offset, read);
+                    _remaining -= result;
+                }
+                else
+                {
+                    result = 0;
+                }
+
+                if (_remaining == 0)
+                {
+                    int cr = _baseStream.ReadByte();
+                    if (cr != '\r')
+                    {
+                        throw new Exception("Expected CR");
+                    }
+                    int lf = _baseStream.ReadByte();
+                    if (lf != '\n')
+                    {
+                        throw new Exception("Expected LF");
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override void SetLength(long value)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            throw new NotImplementedException();
+        }
+
+        private Stream _baseStream;
+        private int _remaining;
+        private bool _eof;
+    }
+}

dotnet/internal/ConsoleCommStruct.cs

@@ -12,7 +12,8 @@ namespace WinSCP
         public uint InputType;
         public uint OutputType;
         public bool WantsProgress; // since version 6
-        public bool UseStdErr; // implies "binary output") since version 10
+        public bool UseStdErr; // since version 10
+        public bool BinaryOutput; // since version 10
         public bool BinaryInput; // since version 10
     }
 

source/console/Console.h

@@ -21,7 +21,8 @@ struct TConsoleCommStruct
     unsigned int InputType;
     unsigned int OutputType;
     bool WantsProgress; // since version 6
-    bool UseStdErr; // (implies "binary output") since version 10
+    bool UseStdErr; // since version 10
+    bool BinaryOutput; // since version 10
     bool BinaryInput; // since version 10
   };
 

source/console/Main.cpp

@@ -788,6 +788,10 @@ inline void ProcessInitEvent(TConsoleCommStruct::TInitEvent& Event)
   if (Event.UseStdErr)
   {
     ConsoleOutput = ConsoleErrorOutput;
+  }
+
+  if (Event.BinaryOutput)
+  {
     setmode(fileno(stdout), O_BINARY);
   }
 

source/core/SftpFileSystem.cpp

@@ -5607,8 +5607,13 @@ void __fastcall TSFTPFileSystem::Sink(
       // queue is discarded here
     }
 
-    if (LocalHandle)
+    if (CopyParam->OnTransferOut != NULL)
     {
+      CopyParam->OnTransferOut(FTerminal, NULL, 0);
+    }
+    else
+    {
+      DebugAssert(LocalHandle);
       if (CopyParam->PreserveTime)
       {
         FTerminal->UpdateTargetTime(LocalHandle, Modification, FTerminal->SessionData->DSTMode);

source/windows/ConsoleRunner.cpp

@@ -523,10 +523,12 @@ UnicodeString __fastcall TOwnConsole::FinalLogMessage()
   return UnicodeString();
 }
 //---------------------------------------------------------------------------
+enum TStdInOutMode { siomOff, siomBinary, siomChunked };
+//---------------------------------------------------------------------------
 class TExternalConsole : public TConsole
 {
 public:
-  __fastcall TExternalConsole(const UnicodeString Instance, bool NoInteractiveInput, bool StdOut, bool StdIn);
+  __fastcall TExternalConsole(const UnicodeString Instance, bool NoInteractiveInput, TStdInOutMode StdOut, TStdInOutMode StdIn);
   virtual __fastcall ~TExternalConsole();
 
   virtual void __fastcall Print(UnicodeString Str, bool FromBeginning = false, bool Error = false);
@@ -553,8 +555,8 @@ private:
   bool FLiveOutput;
   bool FPipeOutput;
   bool FNoInteractiveInput;
-  bool FStdOut;
-  bool FStdIn;
+  TStdInOutMode FStdOut;
+  TStdInOutMode FStdIn;
   bool FWantsProgress;
   bool FInteractive;
   unsigned int FMaxSend;
@@ -564,10 +566,11 @@ private:
   inline void __fastcall SendEvent(int Timeout);
   void __fastcall Init();
   void __fastcall CheckHandle(HANDLE Handle, const UnicodeString & Desc);
+  void __fastcall DoTransferOut(const unsigned char * Data, size_t Len);
 };
 //---------------------------------------------------------------------------
 __fastcall TExternalConsole::TExternalConsole(
-  const UnicodeString Instance, bool NoInteractiveInput, bool StdOut, bool StdIn)
+  const UnicodeString Instance, bool NoInteractiveInput, TStdInOutMode StdOut, TStdInOutMode StdIn)
 {
   UnicodeString Name;
   Name = FORMAT(L"%s%s", (CONSOLE_EVENT_REQUEST, (Instance)));
@@ -825,8 +828,9 @@ void __fastcall TExternalConsole::Init()
   {
     CommStruct->Event = TConsoleCommStruct::INIT;
     CommStruct->InitEvent.WantsProgress = false;
-    CommStruct->InitEvent.UseStdErr = FStdOut;
-    CommStruct->InitEvent.BinaryInput = FStdIn;
+    CommStruct->InitEvent.UseStdErr = (FStdOut != siomOff);
+    CommStruct->InitEvent.BinaryOutput = (FStdOut != siomOff);
+    CommStruct->InitEvent.BinaryInput = (FStdIn != siomOff);
   }
   __finally
   {
@@ -876,10 +880,10 @@ bool __fastcall TExternalConsole::HasFlag(TConsoleFlag Flag) const
       return FWantsProgress;
 
     case cfStdOut:
-      return FStdOut;
+      return (FStdOut != siomOff);
 
     case cfStdIn:
-      return FStdIn;
+      return (FStdIn != siomOff);
 
     default:
       DebugFail();
@@ -958,7 +962,7 @@ void __fastcall TExternalConsole::Progress(TScriptProgress & Progress)
   }
 }
 //---------------------------------------------------------------------------
-void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t Len)
+void __fastcall TExternalConsole::DoTransferOut(const unsigned char * Data, size_t Len)
 {
   size_t Offset = 0;
   while (Offset < Len)
@@ -980,6 +984,38 @@ void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t
   }
 }
 //---------------------------------------------------------------------------
+void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t Len)
+{
+  DebugAssert((Data == NULL) == (Len == 0));
+  if (FStdOut == siomBinary)
+  {
+    if (Data != NULL)
+    {
+      DoTransferOut(Data, Len);
+    }
+  }
+  else if (FStdOut == siomChunked)
+  {
+    std::vector<unsigned char> Buf;
+    // 32 is more than enough for Len in hex + twice CRLF
+    Buf.reserve(Len + 32);
+    // static_cast should not lose digits with IntToHex
+    AnsiString S = AnsiString(IntToHex(static_cast<int>(Len), 0) + L"\r\n");
+    Buf.insert(Buf.end(), S.c_str(), S.c_str() + S.Length());
+    if (Data != NULL) // not really needed
+    {
+      Buf.insert(Buf.end(), Data, Data + Len);
+    }
+    S = "\r\n";
+    Buf.insert(Buf.end(), S.c_str(), S.c_str() + S.Length());
+    DoTransferOut(&Buf.front(), Buf.size());
+  }
+  else
+  {
+    DebugFail();
+  }
+}
+//---------------------------------------------------------------------------
 size_t __fastcall TExternalConsole::TransferIn(unsigned char * Data, size_t Len)
 {
   size_t Offset = 0;
@@ -2301,7 +2337,10 @@ void __fastcall Usage(TConsole * Console)
     FORMAT(L"[/script=<file>] [/%s cmd1...] [/parameter // param1...]", (LowerCase(COMMAND_SWITCH))));
   if (CommandLineOnly)
   {
-    PrintUsageSyntax(Console, FORMAT(L"[/%s] [/%s]", (LowerCase(STDOUT_SWITCH), LowerCase(STDIN_SWITCH))));
+    PrintUsageSyntax(
+      Console, FORMAT(L"[/%s[=%s|%s]] [/%s[=%s|%s]]",
+      (LowerCase(STDOUT_SWITCH), STDINOUT_BINARY_VALUE, STDINOUT_CHUNKED_VALUE,
+       LowerCase(STDIN_SWITCH), STDINOUT_BINARY_VALUE, STDINOUT_CHUNKED_VALUE)));
   }
   PrintUsageSyntax(Console,
     FORMAT(L"[/%s=<logfile> [/loglevel=<level>]] [/%s=[<count>%s]<size>]", (LowerCase(LOG_SWITCH), LowerCase(LOGSIZE_SWITCH), LOGSIZE_SEPARATOR)));
@@ -2807,6 +2846,32 @@ int Info(TConsole * Console)
   return Result;
 }
 //---------------------------------------------------------------------------
+TStdInOutMode ParseStdInOutMode(TProgramParams * Params, const UnicodeString & Switch)
+{
+  TStdInOutMode Result;
+  UnicodeString Value;
+  if (!Params->FindSwitch(Switch, Value))
+  {
+    Result = siomOff;
+  }
+  else
+  {
+    if (Value.IsEmpty() || SameText(Value, STDINOUT_BINARY_VALUE))
+    {
+      Result = siomBinary;
+    }
+    else if (SameText(Value, STDINOUT_CHUNKED_VALUE))
+    {
+      Result = siomChunked;
+    }
+    else
+    {
+      throw Exception(FORMAT(SCRIPT_VALUE_UNKNOWN, (Value, Switch))); // abuse of the string
+    }
+  }
+  return Result;
+}
+//---------------------------------------------------------------------------
 int __fastcall Console(TConsoleMode Mode)
 {
   DebugAssert(Mode != cmNone);
@@ -2823,9 +2888,9 @@ int __fastcall Console(TConsoleMode Mode)
     if (Params->FindSwitch(L"consoleinstance", ConsoleInstance))
     {
       Configuration->Usage->Inc(L"ConsoleExternal");
-      bool StdOut = Params->FindSwitch(STDOUT_SWITCH);
-      bool StdIn = Params->FindSwitch(STDIN_SWITCH);
-      bool NoInteractiveInput = Params->FindSwitch(NOINTERACTIVEINPUT_SWITCH) || StdIn;
+      TStdInOutMode StdOut = ParseStdInOutMode(Params, STDOUT_SWITCH);
+      TStdInOutMode StdIn = ParseStdInOutMode(Params, STDIN_SWITCH);
+      bool NoInteractiveInput = Params->FindSwitch(NOINTERACTIVEINPUT_SWITCH) || (StdIn != siomOff);
       Console = new TExternalConsole(ConsoleInstance, NoInteractiveInput, StdOut, StdIn);
     }
     else if (Params->FindSwitch(L"Console") || (Mode != cmScripting))

source/windows/WinInterface.h

@@ -52,6 +52,8 @@ const int mpAllowContinueOnError = 0x02;
 #define NOINTERACTIVEINPUT_SWITCH L"NoInteractiveInput"
 #define STDOUT_SWITCH L"StdOut"
 #define STDIN_SWITCH L"StdIn"
+#define STDINOUT_BINARY_VALUE L"binary"
+#define STDINOUT_CHUNKED_VALUE L"chunked"
 
 #define DUMPCALLSTACK_EVENT L"WinSCPCallstack%d"