Support reading config from stdin.

Signed-off-by: Daniel Nephin <[email protected]>

compose/cli/command.py

@@ -10,7 +10,7 @@ from .. import config
 from ..project import Project
 from ..service import ConfigError
 from .docopt_command import DocoptCommand
-from .utils import call_silently, is_mac, is_ubuntu, find_candidates_in_parent_dirs
+from .utils import call_silently, is_mac, is_ubuntu
 from .docker_client import docker_client
 from . import verbose_proxy
 from . import errors
@@ -18,13 +18,6 @@ from .. import __version__
 
 log = logging.getLogger(__name__)
 
-SUPPORTED_FILENAMES = [
-    'docker-compose.yml',
-    'docker-compose.yaml',
-    'fig.yml',
-    'fig.yaml',
-]
-
 
 class Command(DocoptCommand):
     base_dir = '.'
@@ -59,7 +52,7 @@ class Command(DocoptCommand):
 
         explicit_config_path = options.get('--file') or os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')
         project = self.get_project(
-            self.get_config_path(explicit_config_path),
+            explicit_config_path,
             project_name=options.get('--project-name'),
             verbose=options.get('--verbose'))
 
@@ -76,16 +69,18 @@ class Command(DocoptCommand):
             return verbose_proxy.VerboseProxy('docker', client)
         return client
 
-    def get_project(self, config_path, project_name=None, verbose=False):
+    def get_project(self, config_path=None, project_name=None, verbose=False):
+        config_details = config.find(self.base_dir, config_path)
+
         try:
             return Project.from_dicts(
-                self.get_project_name(config_path, project_name),
-                config.load(config_path),
+                self.get_project_name(config_details.working_dir, project_name),
+                config.load(config_details),
                 self.get_client(verbose=verbose))
         except ConfigError as e:
             raise errors.UserError(six.text_type(e))
 
-    def get_project_name(self, config_path, project_name=None):
+    def get_project_name(self, working_dir, project_name=None):
         def normalize_name(name):
             return re.sub(r'[^a-z0-9]', '', name.lower())
 
@@ -93,38 +88,15 @@ class Command(DocoptCommand):
             log.warn('The FIG_PROJECT_NAME environment variable is deprecated.')
             log.warn('Please use COMPOSE_PROJECT_NAME instead.')
 
-        project_name = project_name or os.environ.get('COMPOSE_PROJECT_NAME') or os.environ.get('FIG_PROJECT_NAME')
+        project_name = (
+            project_name or
+            os.environ.get('COMPOSE_PROJECT_NAME') or
+            os.environ.get('FIG_PROJECT_NAME'))
         if project_name is not None:
             return normalize_name(project_name)
 
-        project = os.path.basename(os.path.dirname(os.path.abspath(config_path)))
+        project = os.path.basename(os.path.abspath(working_dir))
         if project:
             return normalize_name(project)
 
         return 'default'
-
-    def get_config_path(self, file_path=None):
-        if file_path:
-            return os.path.join(self.base_dir, file_path)
-
-        (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, self.base_dir)
-
-        if len(candidates) == 0:
-            raise errors.ComposeFileNotFound(SUPPORTED_FILENAMES)
-
-        winner = candidates[0]
-
-        if len(candidates) > 1:
-            log.warning("Found multiple config files with supported names: %s", ", ".join(candidates))
-            log.warning("Using %s\n", winner)
-
-        if winner == 'docker-compose.yaml':
-            log.warning("Please be aware that .yml is the expected extension "
-                        "in most cases, and using .yaml can cause compatibility "
-                        "issues in future.\n")
-
-        if winner.startswith("fig."):
-            log.warning("%s is deprecated and will not be supported in future. "
-                        "Please rename your config file to docker-compose.yml\n" % winner)
-
-        return os.path.join(path, winner)

compose/cli/errors.py

@@ -53,12 +53,3 @@ class ConnectionErrorGeneric(UserError):
 
         If it's at a non-standard location, specify the URL with the DOCKER_HOST environment variable.
         """ % url)
-
-
-class ComposeFileNotFound(UserError):
-    def __init__(self, supported_filenames):
-        super(ComposeFileNotFound, self).__init__("""
-        Can't find a suitable configuration file in this directory or any parent. Are you in the right directory?
-
-        Supported filenames: %s
-        """ % ", ".join(supported_filenames))

compose/config.py

@@ -1,7 +1,13 @@
+import logging
 import os
+import sys
 import yaml
+from collections import namedtuple
+
 import six
 
+from compose.cli.utils import find_candidates_in_parent_dirs
+
 
 DOCKER_CONFIG_KEYS = [
     'cap_add',
@@ -64,12 +70,57 @@ DOCKER_CONFIG_HINTS = {
 }
 
 
-def load(filename):
-    working_dir = os.path.dirname(filename)
-    return from_dictionary(load_yaml(filename), working_dir=working_dir, filename=filename)
+SUPPORTED_FILENAMES = [
+    'docker-compose.yml',
+    'docker-compose.yaml',
+    'fig.yml',
+    'fig.yaml',
+]
+
+
+log = logging.getLogger(__name__)
+
+
+ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename')
+
+
+def find(base_dir, filename):
+    if filename == '-':
+        return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None)
+
+    if filename:
+        filename = os.path.join(base_dir, filename)
+    else:
+        filename = get_config_path(base_dir)
+    return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename)
+
+
+def get_config_path(base_dir):
+    (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir)
+
+    if len(candidates) == 0:
+        raise ComposeFileNotFound(SUPPORTED_FILENAMES)
+
+    winner = candidates[0]
+
+    if len(candidates) > 1:
+        log.warn("Found multiple config files with supported names: %s", ", ".join(candidates))
+        log.warn("Using %s\n", winner)
+
+    if winner == 'docker-compose.yaml':
+        log.warn("Please be aware that .yml is the expected extension "
+                 "in most cases, and using .yaml can cause compatibility "
+                 "issues in future.\n")
 
+    if winner.startswith("fig."):
+        log.warn("%s is deprecated and will not be supported in future. "
+                 "Please rename your config file to docker-compose.yml\n" % winner)
 
-def from_dictionary(dictionary, working_dir=None, filename=None):
+    return os.path.join(path, winner)
+
+
+def load(config_details):
+    dictionary, working_dir, filename = config_details
     service_dicts = []
 
     for service_name, service_dict in list(dictionary.items()):
@@ -488,3 +539,12 @@ class CircularReference(ConfigurationError):
             for (filename, service_name) in self.trail
         ]
         return "Circular reference:\n  {}".format("\n  extends ".join(lines))
+
+
+class ComposeFileNotFound(ConfigurationError):
+    def __init__(self, supported_filenames):
+        super(ComposeFileNotFound, self).__init__("""
+        Can't find a suitable configuration file in this directory or any parent. Are you in the right directory?
+
+        Supported filenames: %s
+        """ % ", ".join(supported_filenames))

docs/cli.md

@@ -153,6 +153,9 @@ By default, if there are existing containers for a service, `docker-compose up`
  for `docker-compose.yml` in the current working directory, and then each parent
  directory successively, until found.
 
+ Use a `-` as the filename to read configuration from stdin. When stdin is used
+ all paths in the configuration will be relative to the current working
+ directory.
 
 ### -p, --project-name NAME
 

tests/integration/cli_test.py

@@ -36,7 +36,7 @@ class CLITestCase(DockerClientTestCase):
         if hasattr(self, '_project'):
             return self._project
 
-        return self.command.get_project(self.command.get_config_path())
+        return self.command.get_project()
 
     def test_help(self):
         old_base_dir = self.command.base_dir

tests/integration/project_test.py

@@ -7,6 +7,10 @@ from compose.container import Container
 from .testcases import DockerClientTestCase
 
 
+def build_service_dicts(service_config):
+    return config.load(config.ConfigDetails(service_config, 'working_dir', None))
+
+
 class ProjectTest(DockerClientTestCase):
 
     def test_containers(self):
@@ -32,7 +36,7 @@ class ProjectTest(DockerClientTestCase):
             ['composetest_web_1'])
 
     def test_volumes_from_service(self):
-        service_dicts = config.from_dictionary({
+        service_dicts = build_service_dicts({
             'data': {
                 'image': 'busybox:latest',
                 'volumes': ['/var/data'],
@@ -41,7 +45,7 @@ class ProjectTest(DockerClientTestCase):
                 'image': 'busybox:latest',
                 'volumes_from': ['data'],
             },
-        }, working_dir='.')
+        })
         project = Project.from_dicts(
             name='composetest',
             service_dicts=service_dicts,
@@ -61,7 +65,7 @@ class ProjectTest(DockerClientTestCase):
         )
         project = Project.from_dicts(
             name='composetest',
-            service_dicts=config.from_dictionary({
+            service_dicts=build_service_dicts({
                 'db': {
                     'image': 'busybox:latest',
                     'volumes_from': ['composetest_data_container'],
@@ -75,7 +79,7 @@ class ProjectTest(DockerClientTestCase):
     def test_net_from_service(self):
         project = Project.from_dicts(
             name='composetest',
-            service_dicts=config.from_dictionary({
+            service_dicts=build_service_dicts({
                 'net': {
                     'image': 'busybox:latest',
                     'command': ["top"]
@@ -107,7 +111,7 @@ class ProjectTest(DockerClientTestCase):
 
         project = Project.from_dicts(
             name='composetest',
-            service_dicts=config.from_dictionary({
+            service_dicts=build_service_dicts({
                 'web': {
                     'image': 'busybox:latest',
                     'net': 'container:composetest_net_container'
@@ -274,7 +278,7 @@ class ProjectTest(DockerClientTestCase):
     def test_project_up_starts_depends(self):
         project = Project.from_dicts(
             name='composetest',
-            service_dicts=config.from_dictionary({
+            service_dicts=build_service_dicts({
                 'console': {
                     'image': 'busybox:latest',
                     'command': ["top"],
@@ -309,7 +313,7 @@ class ProjectTest(DockerClientTestCase):
     def test_project_up_with_no_deps(self):
         project = Project.from_dicts(
             name='composetest',
-            service_dicts=config.from_dictionary({
+            service_dicts=build_service_dicts({
                 'console': {
                     'image': 'busybox:latest',
                     'command': ["top"],

tests/integration/state_test.py

@@ -23,7 +23,7 @@ class ProjectTestCase(DockerClientTestCase):
         return Project.from_dicts(
             name='composetest',
             client=self.client,
-            service_dicts=config.from_dictionary(cfg),
+            service_dicts=config.load(config.ConfigDetails(cfg, 'working_dir', None))
         )
 
 

tests/unit/cli_test.py

@@ -2,17 +2,14 @@ from __future__ import unicode_literals
 from __future__ import absolute_import
 import logging
 import os
-import tempfile
-import shutil
 from .. import unittest
 
 import docker
 import mock
 
 from compose.cli import main
-from compose.cli.main import TopLevelCommand
 from compose.cli.docopt_command import NoSuchCommand
-from compose.cli.errors import ComposeFileNotFound
+from compose.cli.main import TopLevelCommand
 from compose.service import Service
 
 
@@ -23,7 +20,7 @@ class CLITestCase(unittest.TestCase):
         try:
             os.chdir('tests/fixtures/simple-composefile')
             command = TopLevelCommand()
-            project_name = command.get_project_name(command.get_config_path())
+            project_name = command.get_project_name('.')
             self.assertEquals('simplecomposefile', project_name)
         finally:
             os.chdir(cwd)
@@ -31,13 +28,13 @@ class CLITestCase(unittest.TestCase):
     def test_project_name_with_explicit_base_dir(self):
         command = TopLevelCommand()
         command.base_dir = 'tests/fixtures/simple-composefile'
-        project_name = command.get_project_name(command.get_config_path())
+        project_name = command.get_project_name(command.base_dir)
         self.assertEquals('simplecomposefile', project_name)
 
     def test_project_name_with_explicit_uppercase_base_dir(self):
         command = TopLevelCommand()
         command.base_dir = 'tests/fixtures/UpperCaseDir'
-        project_name = command.get_project_name(command.get_config_path())
+        project_name = command.get_project_name(command.base_dir)
         self.assertEquals('uppercasedir', project_name)
 
     def test_project_name_with_explicit_project_name(self):
@@ -62,37 +59,10 @@ class CLITestCase(unittest.TestCase):
             project_name = command.get_project_name(None)
         self.assertEquals(project_name, name)
 
-    def test_filename_check(self):
-        files = [
-            'docker-compose.yml',
-            'docker-compose.yaml',
-            'fig.yml',
-            'fig.yaml',
-        ]
-
-        """Test with files placed in the basedir"""
-
-        self.assertEqual('docker-compose.yml', get_config_filename_for_files(files[0:]))
-        self.assertEqual('docker-compose.yaml', get_config_filename_for_files(files[1:]))
-        self.assertEqual('fig.yml', get_config_filename_for_files(files[2:]))
-        self.assertEqual('fig.yaml', get_config_filename_for_files(files[3:]))
-        self.assertRaises(ComposeFileNotFound, lambda: get_config_filename_for_files([]))
-
-        """Test with files placed in the subdir"""
-
-        def get_config_filename_for_files_in_subdir(files):
-            return get_config_filename_for_files(files, subdir=True)
-
-        self.assertEqual('docker-compose.yml', get_config_filename_for_files_in_subdir(files[0:]))
-        self.assertEqual('docker-compose.yaml', get_config_filename_for_files_in_subdir(files[1:]))
-        self.assertEqual('fig.yml', get_config_filename_for_files_in_subdir(files[2:]))
-        self.assertEqual('fig.yaml', get_config_filename_for_files_in_subdir(files[3:]))
-        self.assertRaises(ComposeFileNotFound, lambda: get_config_filename_for_files_in_subdir([]))
-
     def test_get_project(self):
         command = TopLevelCommand()
         command.base_dir = 'tests/fixtures/longer-filename-composefile'
-        project = command.get_project(command.get_config_path())
+        project = command.get_project()
         self.assertEqual(project.name, 'longerfilenamecomposefile')
         self.assertTrue(project.client)
         self.assertTrue(project.services)
@@ -201,23 +171,3 @@ class CLITestCase(unittest.TestCase):
         })
         _, _, call_kwargs = mock_client.create_container.mock_calls[0]
         self.assertFalse('RestartPolicy' in call_kwargs['host_config'])
-
-
-def get_config_filename_for_files(filenames, subdir=None):
-    project_dir = tempfile.mkdtemp()
-    try:
-        make_files(project_dir, filenames)
-        command = TopLevelCommand()
-        if subdir:
-            command.base_dir = tempfile.mkdtemp(dir=project_dir)
-        else:
-            command.base_dir = project_dir
-        return os.path.basename(command.get_config_path())
-    finally:
-        shutil.rmtree(project_dir)
-
-
-def make_files(dirname, filenames):
-    for fname in filenames:
-        with open(os.path.join(dirname, fname), 'w') as f:
-            f.write('')

tests/unit/config_test.py

@@ -1,16 +1,24 @@
-import os
 import mock
+import os
+import shutil
+import tempfile
 from .. import unittest
 
 from compose import config
 
 
 class ConfigTest(unittest.TestCase):
-    def test_from_dictionary(self):
-        service_dicts = config.from_dictionary({
-            'foo': {'image': 'busybox'},
-            'bar': {'environment': ['FOO=1']},
-        })
+    def test_load(self):
+        service_dicts = config.load(
+            config.ConfigDetails(
+                {
+                    'foo': {'image': 'busybox'},
+                    'bar': {'environment': ['FOO=1']},
+                },
+                'working_dir',
+                'filename.yml'
+            )
+        )
 
         self.assertEqual(
             sorted(service_dicts, key=lambda d: d['name']),
@@ -26,11 +34,15 @@ class ConfigTest(unittest.TestCase):
             ])
         )
 
-    def test_from_dictionary_throws_error_when_not_dict(self):
+    def test_load_throws_error_when_not_dict(self):
         with self.assertRaises(config.ConfigurationError):
-            config.from_dictionary({
-                'web': 'busybox:latest',
-            })
+            config.load(
+                config.ConfigDetails(
+                    {'web': 'busybox:latest'},
+                    'working_dir',
+                    'filename.yml'
+                )
+            )
 
     def test_config_validation(self):
         self.assertRaises(
@@ -335,9 +347,13 @@ class EnvTest(unittest.TestCase):
         )
 
 
+def load_from_filename(filename):
+    return config.load(config.find('.', filename))
+
+
 class ExtendsTest(unittest.TestCase):
     def test_extends(self):
-        service_dicts = config.load('tests/fixtures/extends/docker-compose.yml')
+        service_dicts = load_from_filename('tests/fixtures/extends/docker-compose.yml')
 
         service_dicts = sorted(
             service_dicts,
@@ -364,7 +380,7 @@ class ExtendsTest(unittest.TestCase):
         ])
 
     def test_nested(self):
-        service_dicts = config.load('tests/fixtures/extends/nested.yml')
+        service_dicts = load_from_filename('tests/fixtures/extends/nested.yml')
 
         self.assertEqual(service_dicts, [
             {
@@ -380,7 +396,7 @@ class ExtendsTest(unittest.TestCase):
 
     def test_circular(self):
         try:
-            config.load('tests/fixtures/extends/circle-1.yml')
+            load_from_filename('tests/fixtures/extends/circle-1.yml')
             raise Exception("Expected config.CircularReference to be raised")
         except config.CircularReference as e:
             self.assertEqual(
@@ -445,7 +461,7 @@ class ExtendsTest(unittest.TestCase):
             print load_config()
 
     def test_volume_path(self):
-        dicts = config.load('tests/fixtures/volume-path/docker-compose.yml')
+        dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml')
 
         paths = [
             '%s:/foo' % os.path.abspath('tests/fixtures/volume-path/common/foo'),
@@ -455,7 +471,7 @@ class ExtendsTest(unittest.TestCase):
         self.assertEqual(set(dicts[0]['volumes']), set(paths))
 
     def test_parent_build_path_dne(self):
-        child = config.load('tests/fixtures/extends/nonexistent-path-child.yml')
+        child = load_from_filename('tests/fixtures/extends/nonexistent-path-child.yml')
 
         self.assertEqual(child, [
             {
@@ -475,14 +491,16 @@ class BuildPathTest(unittest.TestCase):
         self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx')
 
     def test_nonexistent_path(self):
-        options = {'build': 'nonexistent.path'}
-        self.assertRaises(
-            config.ConfigurationError,
-            lambda: config.from_dictionary({
-                'foo': options,
-                'working_dir': 'tests/fixtures/build-path'
-            })
-        )
+        with self.assertRaises(config.ConfigurationError):
+            config.load(
+                config.ConfigDetails(
+                    {
+                        'foo': {'build': 'nonexistent.path'},
+                    },
+                    'working_dir',
+                    'filename.yml'
+                )
+            )
 
     def test_relative_path(self):
         relative_build_path = '../build-ctx/'
@@ -502,5 +520,56 @@ class BuildPathTest(unittest.TestCase):
         self.assertEquals(service_dict['build'], self.abs_context_path)
 
     def test_from_file(self):
-        service_dict = config.load('tests/fixtures/build-path/docker-compose.yml')
+        service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml')
         self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}])
+
+
+class GetConfigPathTestCase(unittest.TestCase):
+
+    files = [
+        'docker-compose.yml',
+        'docker-compose.yaml',
+        'fig.yml',
+        'fig.yaml',
+    ]
+
+    def test_get_config_path_default_file_in_basedir(self):
+        files = self.files
+        self.assertEqual('docker-compose.yml', get_config_filename_for_files(files[0:]))
+        self.assertEqual('docker-compose.yaml', get_config_filename_for_files(files[1:]))
+        self.assertEqual('fig.yml', get_config_filename_for_files(files[2:]))
+        self.assertEqual('fig.yaml', get_config_filename_for_files(files[3:]))
+        with self.assertRaises(config.ComposeFileNotFound):
+            get_config_filename_for_files([])
+
+    def test_get_config_path_default_file_in_parent_dir(self):
+        """Test with files placed in the subdir"""
+        files = self.files
+
+        def get_config_in_subdir(files):
+            return get_config_filename_for_files(files, subdir=True)
+
+        self.assertEqual('docker-compose.yml', get_config_in_subdir(files[0:]))
+        self.assertEqual('docker-compose.yaml', get_config_in_subdir(files[1:]))
+        self.assertEqual('fig.yml', get_config_in_subdir(files[2:]))
+        self.assertEqual('fig.yaml', get_config_in_subdir(files[3:]))
+        with self.assertRaises(config.ComposeFileNotFound):
+            get_config_in_subdir([])
+
+
+def get_config_filename_for_files(filenames, subdir=None):
+    def make_files(dirname, filenames):
+        for fname in filenames:
+            with open(os.path.join(dirname, fname), 'w') as f:
+                f.write('')
+
+    project_dir = tempfile.mkdtemp()
+    try:
+        make_files(project_dir, filenames)
+        if subdir:
+            base_dir = tempfile.mkdtemp(dir=project_dir)
+        else:
+            base_dir = project_dir
+        return os.path.basename(config.get_config_path(base_dir))
+    finally:
+        shutil.rmtree(project_dir)

tests/unit/project_test.py

@@ -3,7 +3,6 @@ from .. import unittest
 from compose.service import Service
 from compose.project import Project
 from compose.container import Container
-from compose import config
 
 import mock
 import docker
@@ -51,14 +50,16 @@ class ProjectTest(unittest.TestCase):
         self.assertEqual(project.services[2].name, 'web')
 
     def test_from_config(self):
-        dicts = config.from_dictionary({
-            'web': {
+        dicts = [
+            {
+                'name': 'web',
                 'image': 'busybox:latest',
             },
-            'db': {
+            {
+                'name': 'db',
                 'image': 'busybox:latest',
             },
-        })
+        ]
         project = Project.from_dicts('composetest', dicts, None)
         self.assertEqual(len(project.services), 2)
         self.assertEqual(project.get_service('web').name, 'web')