Support multiple config files

Signed-off-by: Daniel Nephin <[email protected]>

compose/cli/command.py

@@ -51,24 +51,26 @@ class Command(DocoptCommand):
             handler(None, command_options)
             return
 
-        if 'FIG_FILE' in os.environ:
-            log.warn('The FIG_FILE environment variable is deprecated.')
-            log.warn('Please use COMPOSE_FILE instead.')
-
-        explicit_config_path = (
-            options.get('--file') or
-            os.environ.get('COMPOSE_FILE') or
-            os.environ.get('FIG_FILE'))
-
         project = get_project(
             self.base_dir,
-            explicit_config_path,
+            get_config_path(options.get('--file')),
             project_name=options.get('--project-name'),
             verbose=options.get('--verbose'))
 
         handler(project, command_options)
 
 
+def get_config_path(file_option):
+    if file_option:
+        return file_option
+
+    if 'FIG_FILE' in os.environ:
+        log.warn('The FIG_FILE environment variable is deprecated.')
+        log.warn('Please use COMPOSE_FILE instead.')
+
+    return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')]
+
+
 def get_client(verbose=False):
     client = docker_client()
     if verbose:

compose/config/config.py

@@ -2,6 +2,7 @@ import logging
 import os
 import sys
 from collections import namedtuple
+from functools import reduce
 
 import six
 import yaml
@@ -88,18 +89,24 @@ PATH_START_CHARS = [
 log = logging.getLogger(__name__)
 
 
-ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename')
+ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs')
 
+ConfigFile = namedtuple('ConfigFile', 'filename config')
 
-def find(base_dir, filename):
-    if filename == '-':
-        return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None)
 
-    if filename:
-        filename = os.path.join(base_dir, filename)
+def find(base_dir, filenames):
+    if filenames == ['-']:
+        return ConfigDetails(
+            os.getcwd(),
+            [ConfigFile(None, yaml.safe_load(sys.stdin))])
+
+    if filenames:
+        filenames = [os.path.join(base_dir, f) for f in filenames]
     else:
-        filename = get_config_path(base_dir)
-    return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename)
+        filenames = [get_config_path(base_dir)]
+    return ConfigDetails(
+        os.path.dirname(filenames[0]),
+        [ConfigFile(f, load_yaml(f)) for f in filenames])
 
 
 def get_config_path(base_dir):
@@ -133,29 +140,40 @@ def pre_process_config(config):
     Pre validation checks and processing of the config file to interpolate env
     vars returning a config dict ready to be tested against the schema.
     """
-    config = interpolate_environment_variables(config)
-    return config
+    return interpolate_environment_variables(config)
 
 
 def load(config_details):
-    config, working_dir, filename = config_details
-
-    processed_config = pre_process_config(config)
-    validate_against_fields_schema(processed_config)
+    working_dir, configs = config_details
 
-    service_dicts = []
-
-    for service_name, service_dict in list(processed_config.items()):
-        loader = ServiceLoader(
-            working_dir=working_dir,
-            filename=filename,
-            service_name=service_name,
-            service_dict=service_dict)
+    def build_service(filename, service_name, service_dict):
+        loader = ServiceLoader(working_dir, filename, service_name, service_dict)
         service_dict = loader.make_service_dict()
         validate_paths(service_dict)
-        service_dicts.append(service_dict)
-
-    return service_dicts
+        return service_dict
+
+    def load_file(filename, config):
+        processed_config = pre_process_config(config)
+        validate_against_fields_schema(processed_config)
+        return [
+            build_service(filename, name, service_config)
+            for name, service_config in processed_config.items()
+        ]
+
+    def merge_services(base, override):
+        return {
+            name: merge_service_dicts(base.get(name, {}), override.get(name, {}))
+            for name in set(base) | set(override)
+        }
+
+    def combine_configs(override, base):
+        service_dicts = load_file(base.filename, base.config)
+        if not override:
+            return service_dicts
+
+        return merge_service_dicts(base.config, override.config)
+
+    return reduce(combine_configs, configs, None)
 
 
 class ServiceLoader(object):

tests/unit/config_test.py

@@ -26,10 +26,16 @@ def service_sort(services):
     return sorted(services, key=itemgetter('name'))
 
 
+def build_config_details(contents, working_dir, filename):
+    return config.ConfigDetails(
+        working_dir,
+        [config.ConfigFile(filename, contents)])
+
+
 class ConfigTest(unittest.TestCase):
     def test_load(self):
         service_dicts = config.load(
-            config.ConfigDetails(
+            build_config_details(
                 {
                     'foo': {'image': 'busybox'},
                     'bar': {'image': 'busybox', 'environment': ['FOO=1']},
@@ -57,7 +63,7 @@ class ConfigTest(unittest.TestCase):
     def test_load_throws_error_when_not_dict(self):
         with self.assertRaises(ConfigurationError):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': 'busybox:latest'},
                     'working_dir',
                     'filename.yml'
@@ -68,7 +74,7 @@ class ConfigTest(unittest.TestCase):
         with self.assertRaises(ConfigurationError):
             for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']:
                 config.load(
-                    config.ConfigDetails(
+                    build_config_details(
                         {invalid_name: {'image': 'busybox'}},
                         'working_dir',
                         'filename.yml'
@@ -79,7 +85,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "Service name: 1 needs to be a string, eg '1'"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {1: {'image': 'busybox'}},
                     'working_dir',
                     'filename.yml'
@@ -89,7 +95,7 @@ class ConfigTest(unittest.TestCase):
     def test_config_valid_service_names(self):
         for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']:
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {valid_name: {'image': 'busybox'}},
                     'tests/fixtures/extends',
                     'common.yml'
@@ -101,7 +107,7 @@ class ConfigTest(unittest.TestCase):
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]:
                 config.load(
-                    config.ConfigDetails(
+                    build_config_details(
                         {'web': {'image': 'busybox', 'ports': invalid_ports}},
                         'working_dir',
                         'filename.yml'
@@ -112,7 +118,7 @@ class ConfigTest(unittest.TestCase):
         valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]]
         for ports in valid_ports:
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {'image': 'busybox', 'ports': ports}},
                     'working_dir',
                     'filename.yml'
@@ -123,7 +129,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "(did you mean 'privileged'?)"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'foo': {'image': 'busybox', 'privilige': 'something'},
                     },
@@ -136,7 +142,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "Service 'foo' has both an image and build path specified."
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'foo': {'image': 'busybox', 'build': '.'},
                     },
@@ -149,7 +155,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'foo': {'image': 'busybox', 'links': 'an_link'},
                     },
@@ -162,7 +168,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "Top level object needs to be a dictionary."
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     ['foo', 'lol'],
                     'tests/fixtures/extends',
                     'filename.yml'
@@ -173,7 +179,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "has non-unique elements"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']}
                     },
@@ -187,7 +193,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg += ", which is an invalid type, it should be a string"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'web': {'build': '.', 'command': [1]}
                     },
@@ -200,7 +206,7 @@ class ConfigTest(unittest.TestCase):
         expected_error_msg = "Service 'web' has both an image and alternate Dockerfile."
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}},
                     'working_dir',
                     'filename.yml'
@@ -212,7 +218,7 @@ class ConfigTest(unittest.TestCase):
 
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {
                         'image': 'busybox',
                         'extra_hosts': 'somehost:162.242.195.82'
@@ -227,7 +233,7 @@ class ConfigTest(unittest.TestCase):
 
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {
                         'image': 'busybox',
                         'extra_hosts': [
@@ -244,7 +250,7 @@ class ConfigTest(unittest.TestCase):
         expose_values = [["8000"], [8000]]
         for expose in expose_values:
             service = config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {
                         'image': 'busybox',
                         'expose': expose
@@ -259,7 +265,7 @@ class ConfigTest(unittest.TestCase):
         entrypoint_values = [["sh"], "sh"]
         for entrypoint in entrypoint_values:
             service = config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {
                         'image': 'busybox',
                         'entrypoint': entrypoint
@@ -331,16 +337,16 @@ class InterpolationTest(unittest.TestCase):
     def test_unset_variable_produces_warning(self):
         os.environ.pop('FOO', None)
         os.environ.pop('BAR', None)
-        config_details = config.ConfigDetails(
-            config={
+        config_details = build_config_details(
+            {
                 'web': {
                     'image': '${FOO}',
                     'command': '${BAR}',
                     'container_name': '${BAR}',
                 },
             },
-            working_dir='.',
-            filename=None,
+            '.',
+            None,
         )
 
         with mock.patch('compose.config.interpolation.log') as log:
@@ -355,7 +361,7 @@ class InterpolationTest(unittest.TestCase):
     def test_invalid_interpolation(self):
         with self.assertRaises(config.ConfigurationError) as cm:
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {'web': {'image': '${'}},
                     'working_dir',
                     'filename.yml'
@@ -371,10 +377,10 @@ class InterpolationTest(unittest.TestCase):
     def test_volume_binding_with_environment_variable(self):
         os.environ['VOLUME_PATH'] = '/host/path'
         d = config.load(
-            config.ConfigDetails(
-                config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
-                working_dir='.',
-                filename=None,
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
+                '.',
+                None,
             )
         )[0]
         self.assertEqual(d['volumes'], ['/host/path:/container/path'])
@@ -649,7 +655,7 @@ class MemoryOptionsTest(unittest.TestCase):
         )
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'foo': {'image': 'busybox', 'memswap_limit': 2000000},
                     },
@@ -660,7 +666,7 @@ class MemoryOptionsTest(unittest.TestCase):
 
     def test_validation_with_correct_memswap_values(self):
         service_dict = config.load(
-            config.ConfigDetails(
+            build_config_details(
                 {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}},
                 'tests/fixtures/extends',
                 'common.yml'
@@ -670,7 +676,7 @@ class MemoryOptionsTest(unittest.TestCase):
 
     def test_memswap_can_be_a_string(self):
         service_dict = config.load(
-            config.ConfigDetails(
+            build_config_details(
                 {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}},
                 'tests/fixtures/extends',
                 'common.yml'
@@ -780,26 +786,26 @@ class EnvTest(unittest.TestCase):
         os.environ['CONTAINERENV'] = '/host/tmp'
 
         service_dict = config.load(
-            config.ConfigDetails(
-                config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
-                working_dir="tests/fixtures/env",
-                filename=None,
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
+                "tests/fixtures/env",
+                None,
             )
         )[0]
         self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp']))
 
         service_dict = config.load(
-            config.ConfigDetails(
-                config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
-                working_dir="tests/fixtures/env",
-                filename=None,
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
+                "tests/fixtures/env",
+                None,
             )
         )[0]
         self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp']))
 
 
 def load_from_filename(filename):
-    return config.load(config.find('.', filename))
+    return config.load(config.find('.', [filename]))
 
 
 class ExtendsTest(unittest.TestCase):
@@ -885,7 +891,7 @@ class ExtendsTest(unittest.TestCase):
     def test_extends_validation_empty_dictionary(self):
         with self.assertRaisesRegexp(ConfigurationError, 'service'):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'web': {'image': 'busybox', 'extends': {}},
                     },
@@ -897,7 +903,7 @@ class ExtendsTest(unittest.TestCase):
     def test_extends_validation_missing_service_key(self):
         with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}},
                     },
@@ -910,7 +916,7 @@ class ExtendsTest(unittest.TestCase):
         expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'web': {
                             'image': 'busybox',
@@ -930,7 +936,7 @@ class ExtendsTest(unittest.TestCase):
         expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type"
         with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'web': {
                             'image': 'busybox',
@@ -955,7 +961,7 @@ class ExtendsTest(unittest.TestCase):
 
     def test_extends_validation_valid_config(self):
         service = config.load(
-            config.ConfigDetails(
+            build_config_details(
                 {
                     'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}},
                 },
@@ -1093,7 +1099,7 @@ class BuildPathTest(unittest.TestCase):
     def test_nonexistent_path(self):
         with self.assertRaises(ConfigurationError):
             config.load(
-                config.ConfigDetails(
+                build_config_details(
                     {
                         'foo': {'build': 'nonexistent.path'},
                     },