Merge pull request #2191 from dnephin/networking

Networking

compose/cli/command.py

@@ -49,7 +49,10 @@ def project_from_options(base_dir, options):
         base_dir,
         get_config_path(options.get('--file')),
         project_name=options.get('--project-name'),
-        verbose=options.get('--verbose'))
+        verbose=options.get('--verbose'),
+        use_networking=options.get('--x-networking'),
+        network_driver=options.get('--x-network-driver'),
+    )
 
 
 def get_config_path(file_option):
@@ -64,8 +67,8 @@ def get_config_path(file_option):
     return [config_file] if config_file else None
 
 
-def get_client(verbose=False):
-    client = docker_client()
+def get_client(verbose=False, version=None):
+    client = docker_client(version=version)
     if verbose:
         version_info = six.iteritems(client.version())
         log.info("Compose version %s", __version__)
@@ -76,14 +79,19 @@ def get_client(verbose=False):
     return client
 
 
-def get_project(base_dir, config_path=None, project_name=None, verbose=False):
+def get_project(base_dir, config_path=None, project_name=None, verbose=False,
+                use_networking=False, network_driver=None):
     config_details = config.find(base_dir, config_path)
 
+    api_version = '1.21' if use_networking else None
     try:
         return Project.from_dicts(
             get_project_name(config_details.working_dir, project_name),
             config.load(config_details),
-            get_client(verbose=verbose))
+            get_client(verbose=verbose, version=api_version),
+            use_networking=use_networking,
+            network_driver=network_driver,
+        )
     except ConfigError as e:
         raise errors.UserError(six.text_type(e))
 

compose/cli/docker_client.py

@@ -9,7 +9,10 @@ from ..const import HTTP_TIMEOUT
 log = logging.getLogger(__name__)
 
 
-def docker_client():
+DEFAULT_API_VERSION = '1.19'
+
+
+def docker_client(version=None):
     """
     Returns a docker-py client configured using environment variables
     according to the same logic as the official Docker client.
@@ -18,6 +21,8 @@ def docker_client():
         log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.')
 
     kwargs = kwargs_from_env(assert_hostname=False)
-    kwargs['version'] = os.environ.get('COMPOSE_API_VERSION', '1.19')
+    kwargs['version'] = version or os.environ.get(
+        'COMPOSE_API_VERSION',
+        DEFAULT_API_VERSION)
     kwargs['timeout'] = HTTP_TIMEOUT
     return Client(**kwargs)

compose/cli/main.py

@@ -117,6 +117,10 @@ class TopLevelCommand(DocoptCommand):
     Options:
       -f, --file FILE           Specify an alternate compose file (default: docker-compose.yml)
       -p, --project-name NAME   Specify an alternate project name (default: directory name)
+      --x-networking            (EXPERIMENTAL) Use new Docker networking functionality.
+                                Requires Docker 1.9 or later.
+      --x-network-driver DRIVER (EXPERIMENTAL) Specify a network driver (default: "bridge").
+                                Requires Docker 1.9 or later.
       --verbose                 Show more output
       -v, --version             Print version and exit
 

compose/project.py

@@ -77,10 +77,12 @@ class Project(object):
     """
     A collection of services.
     """
-    def __init__(self, name, services, client):
+    def __init__(self, name, services, client, use_networking=False, network_driver=None):
         self.name = name
         self.services = services
         self.client = client
+        self.use_networking = use_networking
+        self.network_driver = network_driver or 'bridge'
 
     def labels(self, one_off=False):
         return [
@@ -89,11 +91,15 @@ class Project(object):
         ]
 
     @classmethod
-    def from_dicts(cls, name, service_dicts, client):
+    def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None):
         """
         Construct a ServiceCollection from a list of dicts representing services.
         """
-        project = cls(name, [], client)
+        project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver)
+
+        if use_networking:
+            remove_links(service_dicts)
+
         for service_dict in sort_service_dicts(service_dicts):
             links = project.get_links(service_dict)
             volumes_from = project.get_volumes_from(service_dict)
@@ -103,6 +109,7 @@ class Project(object):
                 Service(
                     client=client,
                     project=name,
+                    use_networking=use_networking,
                     links=links,
                     net=net,
                     volumes_from=volumes_from,
@@ -207,6 +214,8 @@ class Project(object):
     def get_net(self, service_dict):
         net = service_dict.pop('net', None)
         if not net:
+            if self.use_networking:
+                return Net(self.name)
             return Net(None)
 
         net_name = get_service_name_from_net(net)
@@ -289,6 +298,9 @@ class Project(object):
 
         plans = self._get_convergence_plans(services, strategy)
 
+        if self.use_networking:
+            self.ensure_network_exists()
+
         return [
             container
             for service in services
@@ -350,6 +362,26 @@ class Project(object):
 
         return [c for c in containers if matches_service_names(c)]
 
+    def get_network(self):
+        networks = self.client.networks(names=[self.name])
+        if networks:
+            return networks[0]
+        return None
+
+    def ensure_network_exists(self):
+        # TODO: recreate network if driver has changed?
+        if self.get_network() is None:
+            log.info(
+                'Creating network "{}" with driver "{}"'
+                .format(self.name, self.network_driver)
+            )
+            self.client.create_network(self.name, driver=self.network_driver)
+
+    def remove_network(self):
+        network = self.get_network()
+        if network:
+            self.client.remove_network(network['id'])
+
     def _inject_deps(self, acc, service):
         dep_names = service.get_dependency_names()
 
@@ -365,6 +397,26 @@ class Project(object):
         return acc + dep_services
 
 
+def remove_links(service_dicts):
+    services_with_links = [s for s in service_dicts if 'links' in s]
+    if not services_with_links:
+        return
+
+    if len(services_with_links) == 1:
+        prefix = '"{}" defines'.format(services_with_links[0]['name'])
+    else:
+        prefix = 'Some services ({}) define'.format(
+            ", ".join('"{}"'.format(s['name']) for s in services_with_links))
+
+    log.warn(
+        '\n{} links, which are not compatible with Docker networking and will be ignored.\n'
+        'Future versions of Docker will not support links - you should remove them for '
+        'forwards-compatibility.\n'.format(prefix))
+
+    for s in services_with_links:
+        del s['links']
+
+
 class NoSuchService(Exception):
     def __init__(self, name):
         self.name = name

compose/service.py

@@ -113,6 +113,7 @@ class Service(object):
         name,
         client=None,
         project='default',
+        use_networking=False,
         links=None,
         volumes_from=None,
         net=None,
@@ -124,6 +125,7 @@ class Service(object):
         self.name = name
         self.client = client
         self.project = project
+        self.use_networking = use_networking
         self.links = links or []
         self.volumes_from = volumes_from or []
         self.net = net or Net(None)
@@ -602,6 +604,9 @@ class Service(object):
             container_options['hostname'] = parts[0]
             container_options['domainname'] = parts[2]
 
+        if 'hostname' not in container_options and self.use_networking:
+            container_options['hostname'] = self.name
+
         if 'ports' in container_options or 'expose' in self.options:
             ports = []
             all_ports = container_options.get('ports', []) + self.options.get('expose', [])

docs/django.md

@@ -64,9 +64,8 @@ and a `docker-compose.yml` file.
 
     The `docker-compose.yml` file describes the services that make your app. In
     this example those services are a web server and database.  The compose file
-    also describes which Docker images these services use, how they link
-    together, any volumes they might need mounted inside the containers.
-    Finally, the `docker-compose.yml` file describes which ports these services
+    also describes which Docker images these services use, any volumes they might
+    need mounted inside the containers, and any ports they might
     expose. See the [`docker-compose.yml` reference](yml.md) for more
     information on how this file works.
 
@@ -81,8 +80,6 @@ and a `docker-compose.yml` file.
             - .:/code
           ports:
             - "8000:8000"
-          links:
-            - db
 
     This file defines two services: The `db` service and the `web` service.
 

docs/index.md

@@ -128,8 +128,6 @@ Next, define a set of services using `docker-compose.yml`:
        - "5000:5000"
       volumes:
        - .:/code
-      links:
-       - redis
     redis:
       image: redis
 
@@ -138,7 +136,6 @@ This template defines two services, `web` and `redis`. The `web` service:
 * Builds from the `Dockerfile` in the current directory.
 * Forwards the exposed port 5000 on the container to port 5000 on the host machine.
 * Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image.
-* Links the web container to the Redis service.
 
 The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry.
 

docs/networking.md

@@ -0,0 +1,84 @@
+<!--[metadata]>
++++
+title = "Networking in Compose"
+description = "How Compose sets up networking between containers"
+keywords = ["documentation, docs,  docker, compose, orchestration, containers, networking"]
+[menu.main]
+parent="smn_workw_compose"
+weight=6
++++
+<![end-metadata]-->
+
+
+# Networking in Compose
+
+> **Note:** Compose’s networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag.
+
+Compose sets up a single default [network](http://TODO/docker-networking-docs) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name.
+
+> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [CLI docs](cli.md#p-project-name-name) for how to override it.
+
+For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this:
+
+    web:
+      build: .
+      ports:
+        - "8000:8000"
+    db:
+      image: postgres
+
+When you run `docker-compose --x-networking up`, the following happens:
+
+1. A network called `myapp` is created.
+2. A container is created using `web`'s configuration. It joins the network `myapp` under the name `web`.
+3. A container is created using `db`'s configuration. It joins the network `myapp` under the name `db`.
+
+Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s application code could connect to the URL `postgres://db:5432` and start using the Postgres database.
+
+Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface.
+
+## Updating containers
+
+If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working.
+
+If any containers have connections open to the old container, they will be closed. It is a container's responsibility to detect this condition, look up the name again and reconnect.
+
+## Configure how services are published
+
+By default, containers for each service are published on the network with the same name as the service. If you want to change the name, or stop containers from being discoverable at all, you can use the `hostname` option:
+
+    web:
+      build: .
+      hostname: "my-web-application"
+
+This will also change the hostname inside the container, so that the `hostname` command will return `my-web-application`.
+
+## Scaling services
+
+If you create multiple containers for a service with `docker-compose scale`, each container will join the network with the same name. For example, if you run `docker-compose scale web=3`, then 3 containers will join the network under the name `web`. Inside any container on the network, looking up the name `web` will return the IP address of one of them, but Docker and Compose do not provide any guarantees about which one.
+
+This limitation will be addressed in a future version of Compose, where a load balancer will join under the service name and balance traffic between the service's containers in a configurable manner.
+
+## Links
+
+Docker links are a one-way, single-host communication system. They should now be considered deprecated, and you should update your app to use networking instead. In the majority of cases, this will simply involve removing the `links` sections from your `docker-compose.yml`.
+
+## Specifying the network driver
+
+By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](http://TODO/custom-driver-docs).
+
+You can specify which one to use with the `--x-network-driver` flag:
+
+    $ docker-compose --x-networking --x-network-driver=overlay up
+
+## Multi-host networking
+
+(TODO: talk about Swarm and the overlay driver)
+
+## Custom container network modes
+
+Compose allows you to specify a custom network mode for a service with the `net` option - for example, `net: "host"` specifies that its containers should use the same network namespace as the Docker host, and `net: "none"` specifies that they should have no networking capabilities.
+
+If a service specifies the `net` option, its containers will *not* join the app’s network and will not be able to communicate with other services in the app.
+
+If *all* services in an app specify the `net` option, a network will not be created at all.

docs/rails.md

@@ -37,7 +37,7 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten
     source 'https://rubygems.org'
     gem 'rails', '4.2.0'
 
-Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port.
+Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to expose the web app's port.
 
     db:
       image: postgres
@@ -48,8 +48,6 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th
         - .:/myapp
       ports:
         - "3000:3000"
-      links:
-        - db
 
 ### Build the project
 

docs/wordpress.md

@@ -46,8 +46,6 @@ and a separate MySQL instance:
       command: php -S 0.0.0.0:8000 -t /code
       ports:
         - "8000:8000"
-      links:
-        - db
       volumes:
         - .:/code
     db:

requirements.txt

@@ -1,5 +1,5 @@
 PyYAML==3.10
-docker-py==1.4.0
+docker-py==1.5.0
 dockerpty==0.3.4
 docopt==0.6.1
 enum34==1.0.4

script/build-windows.ps1

@@ -42,15 +42,14 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName }
 virtualenv .\venv
 
 # Install dependencies
-.\venv\Scripts\pip install pypiwin32==219
-.\venv\Scripts\pip install -r requirements.txt
-.\venv\Scripts\pip install --no-deps .
-
 # TODO: pip warns when installing from a git sha, so we need to set ErrorAction to
 # 'Continue'.  See
 # https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77
 # This can be removed once pyinstaller 3.x is released and we upgrade 
 $ErrorActionPreference = "Continue"
+.\venv\Scripts\pip install pypiwin32==219
+.\venv\Scripts\pip install -r requirements.txt
+.\venv\Scripts\pip install --no-deps .
 .\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt
 
 # Build binary

script/ci

@@ -6,7 +6,9 @@
 # $ docker build -t "$TAG" .
 # $ docker run --rm --volume="/var/run/docker.sock:/var/run/docker.sock" --volume="$(pwd)/.git:/code/.git" -e "TAG=$TAG" --entrypoint="script/ci" "$TAG"
 
-set -e
+set -ex
+
+docker version
 
 export DOCKER_VERSIONS=all
 export DOCKER_DAEMON_ARGS="--storage-driver=overlay"

setup.py

@@ -33,7 +33,7 @@ install_requires = [
     'requests >= 2.6.1, < 2.8',
     'texttable >= 0.8.1, < 0.9',
     'websocket-client >= 0.32.0, < 1.0',
-    'docker-py >= 1.4.0, < 2',
+    'docker-py >= 1.5.0, < 2',
     'dockerpty >= 0.3.4, < 0.4',
     'six >= 1.3.0, < 2',
     'jsonschema >= 2.5.1, < 3',

tests/integration/cli_test.py

@@ -10,6 +10,7 @@ from six import StringIO
 from .. import mock
 from .testcases import DockerClientTestCase
 from compose.cli.command import get_project
+from compose.cli.docker_client import docker_client
 from compose.cli.errors import UserError
 from compose.cli.main import TopLevelCommand
 from compose.project import NoSuchService
@@ -185,6 +186,51 @@ class CLITestCase(DockerClientTestCase):
             set(self.project.containers())
         )
 
+    def test_up_without_networking(self):
+        self.require_engine_version("1.9")
+
+        self.command.base_dir = 'tests/fixtures/links-composefile'
+        self.command.dispatch(['up', '-d'], None)
+        client = docker_client(version='1.21')
+
+        networks = client.networks(names=[self.project.name])
+        self.assertEqual(len(networks), 0)
+
+        for service in self.project.get_services():
+            containers = service.containers()
+            self.assertEqual(len(containers), 1)
+            self.assertNotEqual(containers[0].get('Config.Hostname'), service.name)
+
+        web_container = self.project.get_service('web').containers()[0]
+        self.assertTrue(web_container.get('HostConfig.Links'))
+
+    def test_up_with_networking(self):
+        self.require_engine_version("1.9")
+
+        self.command.base_dir = 'tests/fixtures/links-composefile'
+        self.command.dispatch(['--x-networking', 'up', '-d'], None)
+        client = docker_client(version='1.21')
+
+        services = self.project.get_services()
+
+        networks = client.networks(names=[self.project.name])
+        for n in networks:
+            self.addCleanup(client.remove_network, n['id'])
+        self.assertEqual(len(networks), 1)
+        self.assertEqual(networks[0]['driver'], 'bridge')
+
+        network = client.inspect_network(networks[0]['id'])
+        self.assertEqual(len(network['containers']), len(services))
+
+        for service in services:
+            containers = service.containers()
+            self.assertEqual(len(containers), 1)
+            self.assertIn(containers[0].id, network['containers'])
+            self.assertEqual(containers[0].get('Config.Hostname'), service.name)
+
+        web_container = self.project.get_service('web').containers()[0]
+        self.assertFalse(web_container.get('HostConfig.Links'))
+
     def test_up_with_links(self):
         self.command.base_dir = 'tests/fixtures/links-composefile'
         self.command.dispatch(['up', '-d', 'web'], None)

tests/integration/testcases.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 from docker import errors
+from docker.utils import version_lt
+from pytest import skip
 
 from .. import unittest
 from compose.cli.docker_client import docker_client
@@ -73,3 +75,12 @@ class DockerClientTestCase(unittest.TestCase):
         kwargs.setdefault('rm', True)
         build_output = self.client.build(*args, **kwargs)
         stream_output(build_output, open('/dev/null', 'w'))
+
+    def require_engine_version(self, minimum):
+        # Drop '-dev' or '-rcN' suffix
+        engine = self.client.version()['Version'].split('-', 1)[0]
+        if version_lt(engine, minimum):
+            skip(
+                "Engine version is too low ({} < {})"
+                .format(engine, minimum)
+            )

tests/unit/service_test.py

@@ -203,6 +203,26 @@ class ServiceTest(unittest.TestCase):
         self.assertEqual(opts['hostname'], 'name.sub', 'hostname')
         self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')
 
+    def test_no_default_hostname_when_not_using_networking(self):
+        service = Service(
+            'foo',
+            image='foo',
+            use_networking=False,
+            client=self.mock_client,
+        )
+        opts = service._get_container_create_options({'image': 'foo'}, 1)
+        self.assertIsNone(opts.get('hostname'))
+
+    def test_hostname_defaults_to_service_name_when_using_networking(self):
+        service = Service(
+            'foo',
+            image='foo',
+            use_networking=True,
+            client=self.mock_client,
+        )
+        opts = service._get_container_create_options({'image': 'foo'}, 1)
+        self.assertEqual(opts['hostname'], 'foo')
+
     def test_get_container_create_options_with_name_option(self):
         service = Service(
             'foo',

tox.ini

@@ -11,9 +11,10 @@ passenv =
 setenv =
     HOME=/tmp
 deps =
+    -rrequirements.txt
     -rrequirements-dev.txt
 commands =
-    py.test -v \
+    py.test -v -rxs \
         --cov=compose \
         --cov-report html \
         --cov-report term \