Add support to CLI build

This includes can be enabled by setting the env var
`COMPOSE_NATIVE_BUILDER=1`.

Signed-off-by: Nao YONASHIRO <[email protected]>

Signed-off-by: Ulysses Souza <[email protected]>

compose/cli/main.py

@@ -263,14 +263,14 @@ class TopLevelCommand(object):
         Usage: build [options] [--build-arg key=val...] [SERVICE...]
 
         Options:
+            --build-arg key=val     Set build-time variables for services.
             --compress              Compress the build context using gzip.
             --force-rm              Always remove intermediate containers.
+            -m, --memory MEM        Sets memory limit for the build container.
             --no-cache              Do not use cache when building the image.
             --no-rm                 Do not remove intermediate containers after a successful build.
-            --pull                  Always attempt to pull a newer version of the image.
-            -m, --memory MEM        Sets memory limit for the build container.
-            --build-arg key=val     Set build-time variables for services.
             --parallel              Build images in parallel.
+            --pull                  Always attempt to pull a newer version of the image.
             -q, --quiet             Don't print anything to STDOUT
         """
         service_names = options['SERVICE']
@@ -283,6 +283,8 @@ class TopLevelCommand(object):
                 )
             build_args = resolve_build_args(build_args, self.toplevel_environment)
 
+        native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER')
+
         self.project.build(
             service_names=options['SERVICE'],
             no_cache=bool(options.get('--no-cache', False)),
@@ -293,7 +295,8 @@ class TopLevelCommand(object):
             build_args=build_args,
             gzip=options.get('--compress', False),
             parallel_build=options.get('--parallel', False),
-            silent=options.get('--quiet', False)
+            silent=options.get('--quiet', False),
+            cli=native_builder,
         )
 
     def bundle(self, options):
@@ -1071,6 +1074,8 @@ class TopLevelCommand(object):
         for excluded in [x for x in opts if options.get(x) and no_start]:
             raise UserError('--no-start and {} cannot be combined.'.format(excluded))
 
+        native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER')
+
         with up_shutdown_context(self.project, service_names, timeout, detached):
             warn_for_swarm_mode(self.project.client)
 
@@ -1090,6 +1095,7 @@ class TopLevelCommand(object):
                     reset_container_image=rebuild,
                     renew_anonymous_volumes=options.get('--renew-anon-volumes'),
                     silent=options.get('--quiet-pull'),
+                    cli=native_builder,
                 )
 
             try:

compose/project.py

@@ -355,7 +355,7 @@ class Project(object):
         return containers
 
     def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None,
-              build_args=None, gzip=False, parallel_build=False, rm=True, silent=False):
+              build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False):
 
         services = []
         for service in self.get_services(service_names):
@@ -364,8 +364,11 @@ class Project(object):
             elif not silent:
                 log.info('%s uses an image, skipping' % service.name)
 
+        if cli:
+            log.warning("Native build is an experimental feature and could change at any time")
+
         def build_service(service):
-            service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent)
+            service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli)
         if parallel_build:
             _, errors = parallel.parallel_execute(
                 services,
@@ -509,8 +512,12 @@ class Project(object):
            reset_container_image=False,
            renew_anonymous_volumes=False,
            silent=False,
+           cli=False,
            ):
 
+        if cli:
+            log.warning("Native build is an experimental feature and could change at any time")
+
         self.initialize()
         if not ignore_orphans:
             self.find_orphan_containers(remove_orphans)
@@ -523,7 +530,7 @@ class Project(object):
             include_deps=start_deps)
 
         for svc in services:
-            svc.ensure_image_exists(do_build=do_build, silent=silent)
+            svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli)
         plans = self._get_convergence_plans(
             services, strategy, always_recreate_deps=always_recreate_deps)
 

compose/service.py

@@ -2,10 +2,13 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import itertools
+import json
 import logging
 import os
 import re
+import subprocess
 import sys
+import tempfile
 from collections import namedtuple
 from collections import OrderedDict
 from operator import attrgetter
@@ -338,9 +341,9 @@ class Service(object):
             raise OperationFailedError("Cannot create container for service %s: %s" %
                                        (self.name, ex.explanation))
 
-    def ensure_image_exists(self, do_build=BuildAction.none, silent=False):
+    def ensure_image_exists(self, do_build=BuildAction.none, silent=False, cli=False):
         if self.can_be_built() and do_build == BuildAction.force:
-            self.build()
+            self.build(cli=cli)
             return
 
         try:
@@ -356,7 +359,7 @@ class Service(object):
         if do_build == BuildAction.skip:
             raise NeedsBuildError(self)
 
-        self.build()
+        self.build(cli=cli)
         log.warning(
             "Image for service {} was built because it did not already exist. To "
             "rebuild this image you must use `docker-compose build` or "
@@ -1049,7 +1052,7 @@ class Service(object):
         return [build_spec(secret) for secret in self.secrets]
 
     def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None,
-              gzip=False, rm=True, silent=False):
+              gzip=False, rm=True, silent=False, cli=False):
         output_stream = open(os.devnull, 'w')
         if not silent:
             output_stream = sys.stdout
@@ -1070,7 +1073,8 @@ class Service(object):
                 'Impossible to perform platform-targeted builds for API version < 1.35'
             )
 
-        build_output = self.client.build(
+        build_image = self.client.build if not cli else cli_build
+        build_output = build_image(
             path=path,
             tag=self.image_name,
             rm=rm,
@@ -1701,3 +1705,124 @@ def rewrite_build_path(path):
         path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path)
 
     return path
+
+
+def cli_build(path, tag=None, quiet=False, fileobj=None,
+              nocache=False, rm=False, timeout=None,
+              custom_context=False, encoding=None, pull=False,
+              forcerm=False, dockerfile=None, container_limits=None,
+              decode=False, buildargs=None, gzip=False, shmsize=None,
+              labels=None, cache_from=None, target=None, network_mode=None,
+              squash=None, extra_hosts=None, platform=None, isolation=None,
+              use_config_proxy=True):
+    """
+    Args:
+        path (str): Path to the directory containing the Dockerfile
+        buildargs (dict): A dictionary of build arguments
+        cache_from (:py:class:`list`): A list of images used for build
+            cache resolution
+        container_limits (dict): A dictionary of limits applied to each
+            container created by the build process. Valid keys:
+            - memory (int): set memory limit for build
+            - memswap (int): Total memory (memory + swap), -1 to disable
+                swap
+            - cpushares (int): CPU shares (relative weight)
+            - cpusetcpus (str): CPUs in which to allow execution, e.g.,
+                ``"0-3"``, ``"0,1"``
+        custom_context (bool): Optional if using ``fileobj``
+        decode (bool): If set to ``True``, the returned stream will be
+            decoded into dicts on the fly. Default ``False``
+        dockerfile (str): path within the build context to the Dockerfile
+        encoding (str): The encoding for a stream. Set to ``gzip`` for
+            compressing
+        extra_hosts (dict): Extra hosts to add to /etc/hosts in building
+            containers, as a mapping of hostname to IP address.
+        fileobj: A file object to use as the Dockerfile. (Or a file-like
+            object)
+        forcerm (bool): Always remove intermediate containers, even after
+            unsuccessful builds
+        isolation (str): Isolation technology used during build.
+            Default: `None`.
+        labels (dict): A dictionary of labels to set on the image
+        network_mode (str): networking mode for the run commands during
+            build
+        nocache (bool): Don't use the cache when set to ``True``
+        platform (str): Platform in the format ``os[/arch[/variant]]``
+        pull (bool): Downloads any updates to the FROM image in Dockerfiles
+        quiet (bool): Whether to return the status
+        rm (bool): Remove intermediate containers. The ``docker build``
+            command now defaults to ``--rm=true``, but we have kept the old
+            default of `False` to preserve backward compatibility
+        shmsize (int): Size of `/dev/shm` in bytes. The size must be
+            greater than 0. If omitted the system uses 64MB
+        squash (bool): Squash the resulting images layers into a
+            single layer.
+        tag (str): A tag to add to the final image
+        target (str): Name of the build-stage to build in a multi-stage
+            Dockerfile
+        timeout (int): HTTP timeout
+        use_config_proxy (bool): If ``True``, and if the docker client
+            configuration file (``~/.docker/config.json`` by default)
+            contains a proxy configuration, the corresponding environment
+            variables will be set in the container being built.
+    Returns:
+        A generator for the build output.
+    """
+    if dockerfile:
+        dockerfile = os.path.join(path, dockerfile)
+    iidfile = tempfile.mktemp()
+
+    command_builder = _CommandBuilder()
+    command_builder.add_params("--build-arg", buildargs)
+    command_builder.add_arg("--file", dockerfile)
+    command_builder.add_flag("--force-rm", forcerm)
+    command_builder.add_arg("--memory", container_limits.get("memory"))
+    command_builder.add_flag("--no-cache", nocache)
+    command_builder.add_flag("--pull", pull)
+    command_builder.add_arg("--iidfile", iidfile)
+    args = command_builder.build([path])
+
+    magic_word = "Successfully built "
+    appear = False
+    with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p:
+        while True:
+            line = p.stdout.readline()
+            if not line:
+                break
+            if line.startswith(magic_word):
+                appear = True
+            yield json.dumps({"stream": line})
+
+    with open(iidfile) as f:
+        line = f.readline()
+        image_id = line.split(":")[1].strip()
+    os.remove(iidfile)
+
+    if not appear:
+        yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)})
+
+
+class _CommandBuilder(object):
+    def __init__(self):
+        self._args = ["docker", "build"]
+
+    def add_arg(self, name, value):
+        if value:
+            self._args.extend([name, str(value)])
+
+    def add_flag(self, name, flag):
+        if flag:
+            self._args.extend([name])
+
+    def add_params(self, name, params):
+        if params:
+            for key, val in params.items():
+                self._args.extend([name, "{}={}".format(key, val)])
+
+    def add_list(self, name, values):
+        if values:
+            for val in values:
+                self._args.extend([name, val])
+
+    def build(self, args):
+        return self._args + args