Merge pull request #255 from orchardup/fix-unicode

Fix encoding errors

.gitignore

@@ -3,4 +3,5 @@
 /build
 /dist
 /docs/_site
+/venv
 fig.spec

fig/cli/log_printer.py

@@ -10,16 +10,17 @@ from .utils import split_buffer
 
 
 class LogPrinter(object):
-    def __init__(self, containers, attach_params=None):
+    def __init__(self, containers, attach_params=None, output=sys.stdout):
         self.containers = containers
         self.attach_params = attach_params or {}
         self.prefix_width = self._calculate_prefix_width(containers)
         self.generators = self._make_log_generators()
+        self.output = output
 
     def run(self):
         mux = Multiplexer(self.generators)
         for line in mux.loop():
-            sys.stdout.write(line.encode(sys.__stdout__.encoding or 'utf-8'))
+            self.output.write(line)
 
     def _calculate_prefix_width(self, containers):
         """
@@ -45,12 +46,12 @@ class LogPrinter(object):
         return generators
 
     def _make_log_generator(self, container, color_fn):
-        prefix = color_fn(self._generate_prefix(container))
+        prefix = color_fn(self._generate_prefix(container)).encode('utf-8')
         # Attach to container before log printer starts running
         line_generator = split_buffer(self._attach(container), '\n')
 
         for line in line_generator:
-            yield prefix + line.decode('utf-8')
+            yield prefix + line
 
         exit_code = container.wait()
         yield color_fn("%s exited with code %s\n" % (container.name, exit_code))

fig/cli/socketclient.py

@@ -81,7 +81,7 @@ class SocketClient:
                 chunk = socket.recv(4096)
 
                 if chunk:
-                    stream.write(chunk.encode(stream.encoding or 'utf-8'))
+                    stream.write(chunk)
                     stream.flush()
                 else:
                     break

fig/progress_stream.py

@@ -0,0 +1,83 @@
+import json
+import os
+import codecs
+
+
+class StreamOutputError(Exception):
+    pass
+
+
+def stream_output(output, stream):
+    is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno())
+    stream = codecs.getwriter('utf-8')(stream)
+    all_events = []
+    lines = {}
+    diff = 0
+
+    for chunk in output:
+        event = json.loads(chunk)
+        all_events.append(event)
+
+        if 'progress' in event or 'progressDetail' in event:
+            image_id = event['id']
+
+            if image_id in lines:
+                diff = len(lines) - lines[image_id]
+            else:
+                lines[image_id] = len(lines)
+                stream.write("\n")
+                diff = 0
+
+            if is_terminal:
+                # move cursor up `diff` rows
+                stream.write("%c[%dA" % (27, diff))
+
+        print_output_event(event, stream, is_terminal)
+
+        if 'id' in event and is_terminal:
+            # move cursor back down
+            stream.write("%c[%dB" % (27, diff))
+
+        stream.flush()
+
+    return all_events
+
+
+def print_output_event(event, stream, is_terminal):
+    if 'errorDetail' in event:
+        raise StreamOutputError(event['errorDetail']['message'])
+
+    terminator = ''
+
+    if is_terminal and 'stream' not in event:
+        # erase current line
+        stream.write("%c[2K\r" % 27)
+        terminator = "\r"
+        pass
+    elif 'progressDetail' in event:
+        return
+
+    if 'time' in event:
+        stream.write("[%s] " % event['time'])
+
+    if 'id' in event:
+        stream.write("%s: " % event['id'])
+
+    if 'from' in event:
+        stream.write("(from %s) " % event['from'])
+
+    status = event.get('status', '')
+
+    if 'progress' in event:
+        stream.write("%s %s%s" % (status, event['progress'], terminator))
+    elif 'progressDetail' in event:
+        detail = event['progressDetail']
+        if 'current' in detail:
+            percentage = float(detail['current']) / float(detail['total']) * 100
+            stream.write('%s (%.1f%%)%s' % (status, percentage, terminator))
+        else:
+            stream.write('%s%s' % (status, terminator))
+    elif 'stream' in event:
+        stream.write("%s%s" % (event['stream'], terminator))
+    else:
+        stream.write("%s%s\n" % (status, terminator))

fig/service.py

@@ -5,8 +5,8 @@ import logging
 import re
 import os
 import sys
-import json
 from .container import Container
+from .progress_stream import stream_output, StreamOutputError
 
 log = logging.getLogger(__name__)
 
@@ -343,84 +343,6 @@ class Service(object):
         return True
 
 
-class StreamOutputError(Exception):
-    pass
-
-
-def stream_output(output, stream):
-    is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno())
-    all_events = []
-    lines = {}
-    diff = 0
-
-    for chunk in output:
-        event = json.loads(chunk)
-        all_events.append(event)
-
-        if 'progress' in event or 'progressDetail' in event:
-            image_id = event['id']
-
-            if image_id in lines:
-                diff = len(lines) - lines[image_id]
-            else:
-                lines[image_id] = len(lines)
-                stream.write("\n")
-                diff = 0
-
-            if is_terminal:
-                # move cursor up `diff` rows
-                stream.write("%c[%dA" % (27, diff))
-
-        print_output_event(event, stream, is_terminal)
-
-        if 'id' in event and is_terminal:
-            # move cursor back down
-            stream.write("%c[%dB" % (27, diff))
-
-        stream.flush()
-
-    return all_events
-
-def print_output_event(event, stream, is_terminal):
-    if 'errorDetail' in event:
-        raise StreamOutputError(event['errorDetail']['message'])
-
-    terminator = ''
-
-    if is_terminal and 'stream' not in event:
-        # erase current line
-        stream.write("%c[2K\r" % 27)
-        terminator = "\r"
-        pass
-    elif 'progressDetail' in event:
-        return
-
-    if 'time' in event:
-        stream.write("[%s] " % event['time'])
-
-    if 'id' in event:
-        stream.write("%s: " % event['id'])
-
-    if 'from' in event:
-        stream.write("(from %s) " % event['from'])
-
-    status = event.get('status', '')
-
-    if 'progress' in event:
-        stream.write("%s %s%s" % (status, event['progress'], terminator))
-    elif 'progressDetail' in event:
-        detail = event['progressDetail']
-        if 'current' in detail:
-            percentage = float(detail['current']) / float(detail['total']) * 100
-            stream.write('%s (%.1f%%)%s' % (status, percentage, terminator))
-        else:
-            stream.write('%s%s' % (status, terminator))
-    elif 'stream' in event:
-        stream.write("%s%s" % (event['stream'], terminator))
-    else:
-        stream.write("%s%s\n" % (status, terminator))
-
-
 NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')
 
 

script/test

@@ -1,2 +1,2 @@
 #!/bin/sh
-nosetests
+PYTHONIOENCODING=ascii nosetests $@

tests/unit/log_printer_test.py

@@ -0,0 +1,57 @@
+from __future__ import unicode_literals
+from __future__ import absolute_import
+import os
+
+from fig.cli.log_printer import LogPrinter
+from .. import unittest
+
+
+class LogPrinterTest(unittest.TestCase):
+    def test_single_container(self):
+        def reader(*args, **kwargs):
+            yield "hello\nworld"
+
+        container = MockContainer(reader)
+        output = run_log_printer([container])
+
+        self.assertIn('hello', output)
+        self.assertIn('world', output)
+
+    def test_unicode(self):
+        glyph = u'\u2022'.encode('utf-8')
+
+        def reader(*args, **kwargs):
+            yield glyph + b'\n'
+
+        container = MockContainer(reader)
+        output = run_log_printer([container])
+
+        self.assertIn(glyph, output)
+
+
+def run_log_printer(containers):
+    r, w = os.pipe()
+    reader, writer = os.fdopen(r, 'r'), os.fdopen(w, 'w')
+    printer = LogPrinter(containers, output=writer)
+    printer.run()
+    writer.close()
+    return reader.read()
+
+
+class MockContainer(object):
+    def __init__(self, reader):
+        self._reader = reader
+
+    @property
+    def name(self):
+        return 'myapp_web_1'
+
+    @property
+    def name_without_project(self):
+        return 'web_1'
+
+    def attach(self, *args, **kwargs):
+        return self._reader()
+
+    def wait(self, *args, **kwargs):
+        return 0

tests/unit/split_buffer_test.py

@@ -6,32 +6,47 @@ from .. import unittest
 class SplitBufferTest(unittest.TestCase):
     def test_single_line_chunks(self):
         def reader():
-            yield "abc\n"
-            yield "def\n"
-            yield "ghi\n"
+            yield b'abc\n'
+            yield b'def\n'
+            yield b'ghi\n'
 
-        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi\n"])
+        self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi\n'])
 
     def test_no_end_separator(self):
         def reader():
-            yield "abc\n"
-            yield "def\n"
-            yield "ghi"
+            yield b'abc\n'
+            yield b'def\n'
+            yield b'ghi'
 
-        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"])
+        self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi'])
 
     def test_multiple_line_chunk(self):
         def reader():
-            yield "abc\ndef\nghi"
+            yield b'abc\ndef\nghi'
 
-        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"])
+        self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi'])
 
     def test_chunked_line(self):
         def reader():
-            yield "a"
-            yield "b"
-            yield "c"
-            yield "\n"
-            yield "d"
+            yield b'a'
+            yield b'b'
+            yield b'c'
+            yield b'\n'
+            yield b'd'
 
-        self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "d"])
+        self.assert_produces(reader, [b'abc\n', b'd'])
+
+    def test_preserves_unicode_sequences_within_lines(self):
+        string = u"a\u2022c\n".encode('utf-8')
+
+        def reader():
+            yield string
+
+        self.assert_produces(reader, [string])
+
+    def assert_produces(self, reader, expectations):
+        split = split_buffer(reader(), b'\n')
+
+        for (actual, expected) in zip(split, expectations):
+            self.assertEqual(type(actual), type(expected))
+            self.assertEqual(actual, expected)