Merge pull request #4616 from dnephin/update-schema-v3.2

Synchronize the schema with docker/docker

compose/config/config.py

@@ -13,11 +13,8 @@ import yaml
 from cached_property import cached_property
 
 from . import types
+from .. import const
 from ..const import COMPOSEFILE_V1 as V1
-from ..const import COMPOSEFILE_V2_0 as V2_0
-from ..const import COMPOSEFILE_V2_1 as V2_1
-from ..const import COMPOSEFILE_V3_0 as V3_0
-from ..const import COMPOSEFILE_V3_1 as V3_1
 from ..utils import build_string_dict
 from ..utils import parse_nanoseconds_int
 from ..utils import splitdrive
@@ -185,10 +182,10 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
                 .format(self.filename, VERSION_EXPLANATION))
 
         if version == '2':
-            version = V2_0
+            version = const.COMPOSEFILE_V2_0
 
         if version == '3':
-            version = V3_0
+            version = const.COMPOSEFILE_V3_0
 
         return version
 
@@ -205,7 +202,7 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
         return {} if self.version == V1 else self.config.get('networks', {})
 
     def get_secrets(self):
-        return {} if self.version < V3_1 else self.config.get('secrets', {})
+        return {} if self.version < const.COMPOSEFILE_V3_1 else self.config.get('secrets', {})
 
 
 class Config(namedtuple('_Config', 'version services volumes networks secrets')):
@@ -427,7 +424,7 @@ def load_services(config_details, config_file):
         service_dict = process_service(resolver.run())
 
         service_config = service_config._replace(config=service_dict)
-        validate_service(service_config, service_names, config_file.version)
+        validate_service(service_config, service_names, config_file)
         service_dict = finalize_service(
             service_config,
             service_names,
@@ -480,7 +477,7 @@ def process_config_file(config_file, environment, service_name=None):
         'service',
         environment)
 
-    if config_file.version in (V2_0, V2_1, V3_0, V3_1):
+    if config_file.version != V1:
         processed_config = dict(config_file.config)
         processed_config['services'] = services
         processed_config['volumes'] = interpolate_config_section(
@@ -493,19 +490,14 @@ def process_config_file(config_file, environment, service_name=None):
             config_file.get_networks(),
             'network',
             environment)
-        if config_file.version in (V3_1,):
+        if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2):
             processed_config['secrets'] = interpolate_config_section(
                 config_file,
                 config_file.get_secrets(),
                 'secrets',
-                environment
-            )
-    elif config_file.version == V1:
-        processed_config = services
+                environment)
     else:
-        raise ConfigurationError(
-            'Version in "{}" is unsupported. {}'
-            .format(config_file.filename, VERSION_EXPLANATION))
+        processed_config = services
 
     config_file = config_file._replace(config=processed_config)
     validate_against_config_schema(config_file)
@@ -642,9 +634,9 @@ def validate_extended_service_dict(service_dict, filename, service):
             "%s services with 'depends_on' cannot be extended" % error_prefix)
 
 
-def validate_service(service_config, service_names, version):
+def validate_service(service_config, service_names, config_file):
     service_dict, service_name = service_config.config, service_config.name
-    validate_service_constraints(service_dict, service_name, version)
+    validate_service_constraints(service_dict, service_name, config_file)
     validate_paths(service_dict)
 
     validate_ulimits(service_config)

compose/config/config_schema_v3.1.json

@@ -71,8 +71,7 @@
               "properties": {
                 "context": {"type": "string"},
                 "dockerfile": {"type": "string"},
-                "args": {"$ref": "#/definitions/list_or_dict"},
-                "cache_from": {"type": "#/definitions/list_of_strings"}
+                "args": {"$ref": "#/definitions/list_or_dict"}
               },
               "additionalProperties": false
             }
@@ -168,21 +167,8 @@
         "ports": {
           "type": "array",
           "items": {
-            "oneOf": [
-              {"type": "number", "format": "ports"},
-              {"type": "string", "format": "ports"},
-              {
-                "type": "object",
-                "properties": {
-                  "mode": {"type": "string"},
-                  "target": {"type": "integer"},
-                  "published": {"type": "integer"},
-                  "protocol": {"type": "string"}
-                },
-                "required": ["target"],
-                "additionalProperties": false
-              }
-            ]
+            "type": ["string", "number"],
+            "format": "ports"
           },
           "uniqueItems": true
         },

compose/config/config_schema_v3.2.json

@@ -0,0 +1,472 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "config_schema_v3.2.json",
+  "type": "object",
+  "required": ["version"],
+
+  "properties": {
+    "version": {
+      "type": "string"
+    },
+
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
+
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "secrets": {
+      "id": "#/properties/secrets",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/secret"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "additionalProperties": false,
+
+  "definitions": {
+
+    "service": {
+      "id": "#/definitions/service",
+      "type": "object",
+
+      "properties": {
+        "deploy": {"$ref": "#/definitions/deployment"},
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"},
+                "cache_from": {"$ref": "#/definitions/list_of_strings"}
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cgroup_parent": {"type": "string"},
+        "command": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "container_name": {"type": "string"},
+        "depends_on": {"$ref": "#/definitions/list_of_strings"},
+        "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "dns": {"$ref": "#/definitions/string_or_list"},
+        "dns_search": {"$ref": "#/definitions/string_or_list"},
+        "domainname": {"type": "string"},
+        "entrypoint": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "env_file": {"$ref": "#/definitions/string_or_list"},
+        "environment": {"$ref": "#/definitions/list_or_dict"},
+
+        "expose": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "expose"
+          },
+          "uniqueItems": true
+        },
+
+        "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+        "healthcheck": {"$ref": "#/definitions/healthcheck"},
+        "hostname": {"type": "string"},
+        "image": {"type": "string"},
+        "ipc": {"type": "string"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+
+        "logging": {
+            "type": "object",
+
+            "properties": {
+                "driver": {"type": "string"},
+                "options": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^.+$": {"type": ["string", "number", "null"]}
+                  }
+                }
+            },
+            "additionalProperties": false
+        },
+
+        "mac_address": {"type": "string"},
+        "network_mode": {"type": "string"},
+
+        "networks": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "aliases": {"$ref": "#/definitions/list_of_strings"},
+                        "ipv4_address": {"type": "string"},
+                        "ipv6_address": {"type": "string"}
+                      },
+                      "additionalProperties": false
+                    },
+                    {"type": "null"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "pid": {"type": ["string", "null"]},
+
+        "ports": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": ["string", "number"], "format": "ports"},
+              {
+                "type": "object",
+                "properties": {
+                  "mode": {"type": "string"},
+                  "target": {"type": "integer"},
+                  "published": {"type": "integer"},
+                  "protocol": {"type": "string"}
+                },
+                "additionalProperties": false
+              }
+            ]
+          },
+          "uniqueItems": true
+        },
+
+        "privileged": {"type": "boolean"},
+        "read_only": {"type": "boolean"},
+        "restart": {"type": "string"},
+        "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "shm_size": {"type": ["number", "string"]},
+        "secrets": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "properties": {
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "uid": {"type": "string"},
+                  "gid": {"type": "string"},
+                  "mode": {"type": "number"}
+                }
+              }
+            ]
+          }
+        },
+        "sysctls": {"$ref": "#/definitions/list_or_dict"},
+        "stdin_open": {"type": "boolean"},
+        "stop_grace_period": {"type": "string", "format": "duration"},
+        "stop_signal": {"type": "string"},
+        "tmpfs": {"$ref": "#/definitions/string_or_list"},
+        "tty": {"type": "boolean"},
+        "ulimits": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "oneOf": [
+                {"type": "integer"},
+                {
+                  "type":"object",
+                  "properties": {
+                    "hard": {"type": "integer"},
+                    "soft": {"type": "integer"}
+                  },
+                  "required": ["soft", "hard"],
+                  "additionalProperties": false
+                }
+              ]
+            }
+          }
+        },
+        "user": {"type": "string"},
+        "userns_mode": {"type": "string"},
+        "volumes": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "required": ["type"],
+                "properties": {
+                  "type": {"type": "string"},
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "read_only": {"type": "boolean"},
+                  "bind": {
+                    "type": "object",
+                    "properties": {
+                      "propagation": {"type": "string"}
+                    }
+                  },
+                  "volume": {
+                    "type": "object",
+                    "properties": {
+                      "nocopy": {"type": "boolean"}
+                    }
+                  }
+                }
+              }
+            ],
+            "uniqueItems": true
+          }
+        },
+        "working_dir": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "healthcheck": {
+      "id": "#/definitions/healthcheck",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "disable": {"type": "boolean"},
+        "interval": {"type": "string"},
+        "retries": {"type": "number"},
+        "test": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "timeout": {"type": "string"}
+      }
+    },
+    "deployment": {
+      "id": "#/definitions/deployment",
+      "type": ["object", "null"],
+      "properties": {
+        "mode": {"type": "string"},
+        "endpoint_mode": {"type": "string"},
+        "replicas": {"type": "integer"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "update_config": {
+          "type": "object",
+          "properties": {
+            "parallelism": {"type": "integer"},
+            "delay": {"type": "string", "format": "duration"},
+            "failure_action": {"type": "string"},
+            "monitor": {"type": "string", "format": "duration"},
+            "max_failure_ratio": {"type": "number"}
+          },
+          "additionalProperties": false
+        },
+        "resources": {
+          "type": "object",
+          "properties": {
+            "limits": {"$ref": "#/definitions/resource"},
+            "reservations": {"$ref": "#/definitions/resource"}
+          }
+        },
+        "restart_policy": {
+          "type": "object",
+          "properties": {
+            "condition": {"type": "string"},
+            "delay": {"type": "string", "format": "duration"},
+            "max_attempts": {"type": "integer"},
+            "window": {"type": "string", "format": "duration"}
+          },
+          "additionalProperties": false
+        },
+        "placement": {
+          "type": "object",
+          "properties": {
+            "constraints": {"type": "array", "items": {"type": "string"}}
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "resource": {
+      "id": "#/definitions/resource",
+      "type": "object",
+      "properties": {
+        "cpus": {"type": "string"},
+        "memory": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "network": {
+      "id": "#/definitions/network",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "ipam": {
+          "type": "object",
+          "properties": {
+            "driver": {"type": "string"},
+            "config": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "subnet": {"type": "string"}
+                },
+                "additionalProperties": false
+              }
+            }
+          },
+          "additionalProperties": false
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "internal": {"type": "boolean"},
+        "attachable": {"type": "boolean"},
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "secret": {
+      "id": "#/definitions/secret",
+      "type": "object",
+      "properties": {
+        "file": {"type": "string"},
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          }
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "string_or_list": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/list_of_strings"}
+      ]
+    },
+
+    "list_of_strings": {
+      "type": "array",
+      "items": {"type": "string"},
+      "uniqueItems": true
+    },
+
+    "list_or_dict": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": ["string", "number", "null"]
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "constraints": {
+      "service": {
+        "id": "#/definitions/constraints/service",
+        "anyOf": [
+          {"required": ["build"]},
+          {"required": ["image"]}
+        ],
+        "properties": {
+          "build": {
+            "required": ["context"]
+          }
+        }
+      }
+    }
+  }
+}

compose/config/serialize.py

@@ -5,9 +5,10 @@ import six
 import yaml
 
 from compose.config import types
-from compose.config.config import V1
-from compose.config.config import V2_1
-from compose.config.config import V3_1
+from compose.const import COMPOSEFILE_V1 as V1
+from compose.const import COMPOSEFILE_V2_1 as V2_1
+from compose.const import COMPOSEFILE_V3_1 as V3_1
+from compose.const import COMPOSEFILE_V3_1 as V3_2
 
 
 def serialize_config_type(dumper, data):
@@ -45,7 +46,7 @@ def denormalize_config(config):
         if 'external_name' in vol_conf:
             del vol_conf['external_name']
 
-    if config.version in (V3_1,):
+    if config.version in (V3_1, V3_2):
         result['secrets'] = config.secrets
     return result
 
@@ -103,7 +104,7 @@ def denormalize_service_dict(service_dict, version):
                 service_dict['healthcheck']['timeout']
             )
 
-    if 'ports' in service_dict and version != V3_1:
+    if 'ports' in service_dict and version not in (V3_2,):
         service_dict['ports'] = map(
             lambda p: p.legacy_repr() if isinstance(p, types.ServicePort) else p,
             service_dict['ports']

compose/config/validation.py

@@ -365,7 +365,7 @@ def process_config_schema_errors(error):
 
 
 def validate_against_config_schema(config_file):
-    schema = load_jsonschema(config_file.version)
+    schema = load_jsonschema(config_file)
     format_checker = FormatChecker(["ports", "expose"])
     validator = Draft4Validator(
         schema,
@@ -377,11 +377,12 @@ def validate_against_config_schema(config_file):
         config_file.filename)
 
 
-def validate_service_constraints(config, service_name, version):
+def validate_service_constraints(config, service_name, config_file):
     def handler(errors):
-        return process_service_constraint_errors(errors, service_name, version)
+        return process_service_constraint_errors(
+            errors, service_name, config_file.version)
 
-    schema = load_jsonschema(version)
+    schema = load_jsonschema(config_file)
     validator = Draft4Validator(schema['definitions']['constraints']['service'])
     handle_errors(validator.iter_errors(config), handler, None)
 
@@ -390,10 +391,15 @@ def get_schema_path():
     return os.path.dirname(os.path.abspath(__file__))
 
 
-def load_jsonschema(version):
+def load_jsonschema(config_file):
     filename = os.path.join(
         get_schema_path(),
-        "config_schema_v{0}.json".format(version))
+        "config_schema_v{0}.json".format(config_file.version))
+
+    if not os.path.exists(filename):
+        raise ConfigurationError(
+            'Version in "{}" is unsupported. {}'
+            .format(config_file.filename, VERSION_EXPLANATION))
 
     with open(filename, "r") as fh:
         return json.load(fh)

compose/const.py

@@ -21,8 +21,10 @@ SECRETS_PATH = '/run/secrets'
 COMPOSEFILE_V1 = '1'
 COMPOSEFILE_V2_0 = '2.0'
 COMPOSEFILE_V2_1 = '2.1'
+
 COMPOSEFILE_V3_0 = '3.0'
 COMPOSEFILE_V3_1 = '3.1'
+COMPOSEFILE_V3_2 = '3.2'
 
 API_VERSIONS = {
     COMPOSEFILE_V1: '1.21',
@@ -30,6 +32,7 @@ API_VERSIONS = {
     COMPOSEFILE_V2_1: '1.24',
     COMPOSEFILE_V3_0: '1.25',
     COMPOSEFILE_V3_1: '1.25',
+    COMPOSEFILE_V3_2: '1.25',
 }
 
 API_VERSION_TO_ENGINE_VERSION = {
@@ -38,4 +41,5 @@ API_VERSION_TO_ENGINE_VERSION = {
     API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0',
     API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0',
     API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0',
+    API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0',
 }

tests/fixtures/ports-composefile/expanded-notation.yml

@@ -1,4 +1,4 @@
-version: '3.1'
+version: '3.2'
 services:
     simple:
       image: busybox:latest

tests/integration/project_test.py

@@ -15,11 +15,11 @@ from .testcases import DockerClientTestCase
 from compose.config import config
 from compose.config import ConfigurationError
 from compose.config import types
-from compose.config.config import V2_0
-from compose.config.config import V2_1
-from compose.config.config import V3_1
 from compose.config.types import VolumeFromSpec
 from compose.config.types import VolumeSpec
+from compose.const import COMPOSEFILE_V2_0 as V2_0
+from compose.const import COMPOSEFILE_V2_1 as V2_1
+from compose.const import COMPOSEFILE_V3_1 as V3_1
 from compose.const import LABEL_PROJECT
 from compose.const import LABEL_SERVICE
 from compose.container import Container

tests/integration/testcases.py

@@ -10,12 +10,12 @@ from pytest import skip
 from .. import unittest
 from compose.cli.docker_client import docker_client
 from compose.config.config import resolve_environment
-from compose.config.config import V1
-from compose.config.config import V2_0
-from compose.config.config import V2_1
-from compose.config.config import V3_0
 from compose.config.environment import Environment
 from compose.const import API_VERSIONS
+from compose.const import COMPOSEFILE_V1 as V1
+from compose.const import COMPOSEFILE_V2_0 as V2_0
+from compose.const import COMPOSEFILE_V2_0 as V2_1
+from compose.const import COMPOSEFILE_V3_0 as V3_0
 from compose.const import LABEL_PROJECT
 from compose.progress_stream import stream_output
 from compose.service import Service

tests/unit/config/config_test.py

@@ -17,11 +17,6 @@ from compose.config import config
 from compose.config import types
 from compose.config.config import resolve_build_args
 from compose.config.config import resolve_environment
-from compose.config.config import V1
-from compose.config.config import V2_0
-from compose.config.config import V2_1
-from compose.config.config import V3_0
-from compose.config.config import V3_1
 from compose.config.environment import Environment
 from compose.config.errors import ConfigurationError
 from compose.config.errors import VERSION_EXPLANATION
@@ -29,6 +24,11 @@ from compose.config.serialize import denormalize_service_dict
 from compose.config.serialize import serialize_config
 from compose.config.serialize import serialize_ns_time_value
 from compose.config.types import VolumeSpec
+from compose.const import COMPOSEFILE_V1 as V1
+from compose.const import COMPOSEFILE_V2_0 as V2_0
+from compose.const import COMPOSEFILE_V2_1 as V2_1
+from compose.const import COMPOSEFILE_V3_0 as V3_0
+from compose.const import COMPOSEFILE_V3_1 as V3_1
 from compose.const import IS_WINDOWS_PLATFORM
 from compose.utils import nanoseconds_from_time_seconds
 from tests import mock

tests/unit/config/types_test.py

@@ -3,13 +3,13 @@ from __future__ import unicode_literals
 
 import pytest
 
-from compose.config.config import V1
-from compose.config.config import V2_0
 from compose.config.errors import ConfigurationError
 from compose.config.types import parse_extra_hosts
 from compose.config.types import ServicePort
 from compose.config.types import VolumeFromSpec
 from compose.config.types import VolumeSpec
+from compose.const import COMPOSEFILE_V1 as V1
+from compose.const import COMPOSEFILE_V2_0 as V2_0
 
 
 def test_parse_extra_hosts_list():