Merge pull request #4604 from shin-/4584-interpolate-secrets

Interpolate secrets

compose/config/config.py

@@ -218,6 +218,8 @@ class Config(namedtuple('_Config', 'version services volumes networks secrets'))
     :type  volumes: :class:`dict`
     :param networks: Dictionary mapping network names to description dictionaries
     :type  networks: :class:`dict`
+    :param secrets: Dictionary mapping secret names to description dictionaries
+    :type secrets: :class:`dict`
     """
 
 
@@ -491,6 +493,13 @@ def process_config_file(config_file, environment, service_name=None):
             config_file.get_networks(),
             'network',
             environment)
+        if config_file.version in (V3_1,):
+            processed_config['secrets'] = interpolate_config_section(
+                config_file,
+                config_file.get_secrets(),
+                'secrets',
+                environment
+            )
     elif config_file.version == V1:
         processed_config = services
     else:

compose/config/errors.py

@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 
 VERSION_EXPLANATION = (
     'You might be seeing this error because you\'re using the wrong Compose file version. '
-    'Either specify a supported version ("2.0", "2.1", "3.0") and place your '
+    'Either specify a supported version ("2.0", "2.1", "3.0", "3.1") and place your '
     'service definitions under the `services` key, or omit the `version` key '
     'and place your service definitions at the root of the file to use '
     'version 1.\nFor more on the Compose file format versions, see '

compose/config/serialize.py

@@ -26,34 +26,28 @@ yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type)
 
 
 def denormalize_config(config):
+    result = {'version': V2_1 if config.version == V1 else config.version}
     denormalized_services = [
         denormalize_service_dict(service_dict, config.version)
         for service_dict in config.services
     ]
-    services = {
+    result['services'] = {
         service_dict.pop('name'): service_dict
         for service_dict in denormalized_services
     }
-    networks = config.networks.copy()
-    for net_name, net_conf in networks.items():
+    result['networks'] = config.networks.copy()
+    for net_name, net_conf in result['networks'].items():
         if 'external_name' in net_conf:
             del net_conf['external_name']
 
-    volumes = config.volumes.copy()
-    for vol_name, vol_conf in volumes.items():
+    result['volumes'] = config.volumes.copy()
+    for vol_name, vol_conf in result['volumes'].items():
         if 'external_name' in vol_conf:
             del vol_conf['external_name']
 
-    version = config.version
-    if version == V1:
-        version = V2_1
-
-    return {
-        'version': version,
-        'services': services,
-        'networks': networks,
-        'volumes': volumes,
-    }
+    if config.version in (V3_1,):
+        result['secrets'] = config.secrets
+    return result
 
 
 def serialize_config(config):

tests/unit/config/config_test.py

@@ -1821,6 +1821,23 @@ class ConfigTest(unittest.TestCase):
             }
         }
 
+    def test_empty_environment_key_allowed(self):
+        service_dict = config.load(
+            build_config_details(
+                {
+                    'web': {
+                        'build': '.',
+                        'environment': {
+                            'POSTGRES_PASSWORD': ''
+                        },
+                    },
+                },
+                '.',
+                None,
+            )
+        ).services[0]
+        self.assertEqual(service_dict['environment']['POSTGRES_PASSWORD'], '')
+
     def test_merge_pid(self):
         # Regression: https://github.com/docker/compose/issues/4184
         base = {
@@ -2335,22 +2352,23 @@ class InterpolationTest(unittest.TestCase):
         self.assertIn('in service "web"', cm.exception.msg)
         self.assertIn('"${"', cm.exception.msg)
 
-    def test_empty_environment_key_allowed(self):
-        service_dict = config.load(
-            build_config_details(
-                {
-                    'web': {
-                        'build': '.',
-                        'environment': {
-                            'POSTGRES_PASSWORD': ''
-                        },
-                    },
-                },
-                '.',
-                None,
-            )
-        ).services[0]
-        self.assertEqual(service_dict['environment']['POSTGRES_PASSWORD'], '')
+    @mock.patch.dict(os.environ)
+    def test_interpolation_secrets_section(self):
+        os.environ['FOO'] = 'baz.bar'
+        config_dict = config.load(build_config_details({
+            'version': '3.1',
+            'secrets': {
+                'secretdata': {
+                    'external': {'name': '$FOO'}
+                }
+            }
+        }))
+        assert config_dict.secrets == {
+            'secretdata': {
+                'external': {'name': 'baz.bar'},
+                'external_name': 'baz.bar'
+            }
+        }
 
 
 class VolumeConfigTest(unittest.TestCase):
@@ -3650,11 +3668,17 @@ class SerializeTest(unittest.TestCase):
                 }
             ]
         }
+        secrets_dict = {
+            'one': {'file': '/one.txt'},
+            'source': {'file': '/source.pem'}
+        }
         config_dict = config.load(build_config_details({
             'version': '3.1',
-            'services': {'web': service_dict}
+            'services': {'web': service_dict},
+            'secrets': secrets_dict
         }))
 
         serialized_config = yaml.load(serialize_config(config_dict))
         serialized_service = serialized_config['services']['web']
         assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets'])
+        assert 'secrets' in serialized_config

tests/unit/config/interpolation_test.py

@@ -75,7 +75,32 @@ def test_interpolate_environment_variables_in_volumes(mock_env):
         },
         'other': {},
     }
-    value = interpolate_environment_variables("2.0",  volumes, 'volume', mock_env)
+    value = interpolate_environment_variables("2.0", volumes, 'volume', mock_env)
+    assert value == expected
+
+
+def test_interpolate_environment_variables_in_secrets(mock_env):
+    secrets = {
+        'secretservice': {
+            'file': '$FOO',
+            'labels': {
+                'max': 2,
+                'user': '${USER}'
+            }
+        },
+        'other': None,
+    }
+    expected = {
+        'secretservice': {
+            'file': 'bar',
+            'labels': {
+                'max': 2,
+                'user': 'jenny'
+            }
+        },
+        'other': {},
+    }
+    value = interpolate_environment_variables("3.1", secrets, 'volume', mock_env)
     assert value == expected