Merge pull request #6255 from docker/bump-1.23.0-rc2

Bump 1.23.0-rc2

CHANGELOG.md

@@ -56,6 +56,12 @@ naming scheme accordingly before upgrading.
 - Fixed a bug causing `external: false` entries in the Compose file to be
   printed as `external: true` in the output of `docker-compose config`
 
+- Fixed a bug where issuing a `docker-compose pull` command on services
+  without a defined image key would cause Compose to crash
+
+- Volumes and binds are now mounted in the order they're declared in the
+  service definition
+
 ### Miscellaneous
 
 - The `zsh` completion script has been updated with new options, and no

compose/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-__version__ = '1.23.0-rc1'
+__version__ = '1.23.0-rc2'

compose/project.py

@@ -34,6 +34,7 @@ from .service import Service
 from .service import ServiceNetworkMode
 from .service import ServicePidMode
 from .utils import microseconds_from_time_nano
+from .utils import truncate_string
 from .volume import ProjectVolumes
 
 
@@ -554,12 +555,10 @@ class Project(object):
         if parallel_pull:
             def pull_service(service):
                 strm = service.pull(ignore_pull_failures, True, stream=True)
-                writer = parallel.get_stream_writer()
+                if strm is None:  # Attempting to pull service with no `image` key is a no-op
+                    return
 
-                def trunc(s):
-                    if len(s) > 35:
-                        return s[:33] + '...'
-                    return s
+                writer = parallel.get_stream_writer()
 
                 for event in strm:
                     if 'status' not in event:
@@ -572,7 +571,7 @@ class Project(object):
                             status = '{} ({:.1%})'.format(status, percentage)
 
                     writer.write(
-                        msg, service.name, trunc(status), lambda s: s
+                        msg, service.name, truncate_string(status), lambda s: s
                     )
 
             _, errors = parallel.parallel_execute(

compose/service.py

@@ -56,6 +56,7 @@ from .utils import json_hash
 from .utils import parse_bytes
 from .utils import parse_seconds_float
 from .utils import truncate_id
+from .utils import unique_everseen
 
 
 log = logging.getLogger(__name__)
@@ -940,8 +941,9 @@ class Service(object):
                 override_options['mounts'] = override_options.get('mounts') or []
                 override_options['mounts'].extend([build_mount(v) for v in secret_volumes])
 
-        # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885)
-        override_options['binds'] = list(set(binds))
+        # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885).
+        # unique_everseen preserves order. (see https://github.com/docker/compose/issues/6091).
+        override_options['binds'] = list(unique_everseen(binds))
         return container_options, override_options
 
     def _get_container_host_config(self, override_options, one_off=False):
@@ -1427,7 +1429,7 @@ def merge_volume_bindings(volumes, tmpfs, previous_container, mounts):
     """
     affinity = {}
 
-    volume_bindings = dict(
+    volume_bindings = OrderedDict(
         build_volume_binding(volume)
         for volume in volumes
         if volume.external

compose/utils.py

@@ -170,3 +170,19 @@ def truncate_id(value):
     if len(value) > 12:
         return value[:12]
     return value
+
+
+def unique_everseen(iterable, key=lambda x: x):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    seen = set()
+    for element in iterable:
+        unique_key = key(element)
+        if unique_key not in seen:
+            seen.add(unique_key)
+            yield element
+
+
+def truncate_string(s, max_chars=35):
+    if len(s) > max_chars:
+        return s[:max_chars - 2] + '...'
+    return s

contrib/completion/bash/docker-compose

@@ -136,7 +136,18 @@ _docker_compose_bundle() {
 
 
 _docker_compose_config() {
-	COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes --hash" -- "$cur" ) )
+	case "$prev" in
+		--hash)
+			if [[ $cur == \\* ]] ; then
+				COMPREPLY=( '\*' )
+			else
+				COMPREPLY=( $(compgen -W "$(__docker_compose_services) \\\* " -- "$cur") )
+			fi
+			return
+			;;
+	esac
+
+	COMPREPLY=( $( compgen -W "--hash --help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) )
 }
 
 

script/release/release.py

@@ -173,9 +173,10 @@ def distclean():
 def pypi_upload(args):
     print('Uploading to PyPi')
     try:
+        rel = args.release.replace('-rc', 'rc')
         twine_upload([
-            'dist/docker_compose-{}*.whl'.format(args.release),
-            'dist/docker-compose-{}*.tar.gz'.format(args.release)
+            'dist/docker_compose-{}*.whl'.format(rel),
+            'dist/docker-compose-{}*.tar.gz'.format(rel)
         ])
     except HTTPError as e:
         if e.response.status_code == 400 and 'File already exists' in e.message:

script/run/run.sh

@@ -15,7 +15,7 @@
 
 set -e
 
-VERSION="1.23.0-rc1"
+VERSION="1.23.0-rc2"
 IMAGE="docker/compose:$VERSION"
 
 

script/test/versions.py

@@ -36,6 +36,8 @@ import requests
 
 GITHUB_API = 'https://api.github.com/repos'
 
+STAGES = ['tp', 'beta', 'rc']
+
 
 class Version(namedtuple('_Version', 'major minor patch stage edition')):
 
@@ -45,7 +47,7 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')):
         version = version.lstrip('v')
         version, _, stage = version.partition('-')
         if stage:
-            if not any(marker in stage for marker in ['rc', 'tp', 'beta']):
+            if not any(marker in stage for marker in STAGES):
                 edition = stage
                 stage = None
             elif '-' in stage:
@@ -62,8 +64,16 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')):
         """Return a representation that allows this object to be sorted
         correctly with the default comparator.
         """
-        # rc releases should appear before official releases
-        stage = (0, self.stage) if self.stage else (1, )
+        # non-GA releases should appear before GA releases
+        # Order: tp -> beta -> rc -> GA
+        if self.stage:
+            for st in STAGES:
+                if st in self.stage:
+                    stage = (STAGES.index(st), self.stage)
+                    break
+        else:
+            stage = (len(STAGES),)
+
         return (int(self.major), int(self.minor), int(self.patch)) + stage
 
     def __str__(self):
@@ -124,9 +134,6 @@ def get_versions(tags):
             v = Version.parse(tag['name'])
             if v in BLACKLIST:
                 continue
-            # FIXME: Temporary. Remove once these versions are built on dockerswarm/dind
-            if v.stage and 'rc' not in v.stage:
-                continue
             yield v
         except ValueError:
             print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr)

tests/integration/project_test.py

@@ -105,6 +105,23 @@ class ProjectTest(DockerClientTestCase):
         project = Project('composetest', [web, db], self.client)
         assert set(project.containers(stopped=True)) == set([web_1, db_1])
 
+    def test_parallel_pull_with_no_image(self):
+        config_data = build_config(
+            version=V2_3,
+            services=[{
+                'name': 'web',
+                'build': {'context': '.'},
+            }],
+        )
+
+        project = Project.from_config(
+            name='composetest',
+            config_data=config_data,
+            client=self.client
+        )
+
+        project.pull(parallel_pull=True)
+
     def test_volumes_from_service(self):
         project = Project.from_config(
             name='composetest',

tests/unit/config/config_test.py

@@ -8,6 +8,7 @@ import os
 import shutil
 import tempfile
 from operator import itemgetter
+from random import shuffle
 
 import py
 import pytest
@@ -42,7 +43,7 @@ from tests import unittest
 DEFAULT_VERSION = V2_0
 
 
-def make_service_dict(name, service_dict, working_dir, filename=None):
+def make_service_dict(name, service_dict, working_dir='.', filename=None):
     """Test helper function to construct a ServiceExtendsResolver
     """
     resolver = config.ServiceExtendsResolver(
@@ -3536,6 +3537,13 @@ class VolumeConfigTest(unittest.TestCase):
         ).services[0]
         assert d['volumes'] == [VolumeSpec.parse('/host/path:/container/path')]
 
+    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths')
+    def test_volumes_order_is_preserved(self):
+        volumes = ['/{0}:/{0}'.format(i) for i in range(0, 6)]
+        shuffle(volumes)
+        cfg = make_service_dict('foo', {'build': '.', 'volumes': volumes})
+        assert cfg['volumes'] == volumes
+
     @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths')
     @mock.patch.dict(os.environ)
     def test_volume_binding_with_home(self):

tests/unit/service_test.py

@@ -1037,6 +1037,23 @@ class ServiceTest(unittest.TestCase):
         assert len(override_opts['binds']) == 1
         assert override_opts['binds'][0] == 'vol:/data:rw'
 
+    def test_volumes_order_is_preserved(self):
+        service = Service('foo', client=self.mock_client)
+        volumes = [
+            VolumeSpec.parse(cfg) for cfg in [
+                '/v{0}:/v{0}:rw'.format(i) for i in range(6)
+            ]
+        ]
+        ctnr_opts, override_opts = service._build_container_volume_options(
+            previous_container=None,
+            container_options={
+                'volumes': volumes,
+                'environment': {},
+            },
+            override_options={},
+        )
+        assert override_opts['binds'] == [vol.repr() for vol in volumes]
+
 
 class TestServiceNetwork(unittest.TestCase):
     def setUp(self):

tests/unit/utils_test.py

@@ -68,3 +68,11 @@ class TestParseBytes(object):
         assert utils.parse_bytes(123) == 123
         assert utils.parse_bytes('foobar') is None
         assert utils.parse_bytes('123') == 123
+
+
+class TestMoreItertools(object):
+    def test_unique_everseen(self):
+        unique = utils.unique_everseen
+        assert list(unique([2, 1, 2, 1])) == [2, 1]
+        assert list(unique([2, 1, 2, 1], hash)) == [2, 1]
+        assert list(unique([2, 1, 2, 1], lambda x: 'key_%s' % x)) == [2, 1]