Merge back changes made in docker/contrib

README.md

@@ -8,7 +8,7 @@ docker-brew is a command-line tool used to build the docker standard library.
 1. Install the easy_install tool (`sudo apt-get install python-setuptools`
 for Debian)
 1. Install the python package manager, `pip` (`easy_install pip`)
-1. Run the following command: `pip install -r requirements.txt`
+1. Run the following command: `sudo pip install -r requirements.txt`
 1. You should now be able to use the `docker-brew` script as such.
 
 ## Basics

brew/__init__.py

@@ -1 +1 @@
-from brew import build_library
+from brew import build_library, DEFAULT_REPOSITORY, DEFAULT_BRANCH

brew/brew.py

@@ -1,36 +1,71 @@
 import os
 import logging
+from shutil import rmtree
 
 import docker
 
 import git
 
 DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker'
-DEFAULT_BRANCH = 'library'
+DEFAULT_BRANCH = 'master'
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
-                    level='DEBUG')
+                    level='INFO')
 client = docker.Client()
 processed = {}
+processed_folders = []
 
 
-def build_library(repository=None, branch=None, namespace=None, push=False):
+def build_library(repository=None, branch=None, namespace=None, push=False,
+        debug=False, prefill=True, registry=None):
+    dst_folder = None
+    summary = Summary()
     if repository is None:
         repository = DEFAULT_REPOSITORY
     if branch is None:
         branch = DEFAULT_BRANCH
+    if debug:
+        logger.setLevel('DEBUG')
+
+    if not (repository.startswith('https://') or repository.startswith('git://')):
+        logger.info('Repository provided assumed to be a local path')
+        dst_folder = repository
+
+    try:
+        client.version()
+    except Exception as e:
+        logger.error('Could not reach the docker daemon. Please make sure it '
+            'is running.')
+        logger.warning('Also make sure you have access to the docker UNIX '
+            'socket (use sudo)')
+        return
 
-    logger.info('Cloning docker repo from {0}, branch: {1}'.format(
-        repository, branch))
     #FIXME: set destination folder and only pull latest changes instead of
     # cloning the whole repo everytime
-    dst_folder = git.clone_branch(repository, branch)
-    for buildfile in os.listdir(os.path.join(dst_folder, 'library')):
+    if not dst_folder:
+        logger.info('Cloning docker repo from {0}, branch: {1}'.format(
+            repository, branch))
+        try:
+            rep, dst_folder = git.clone_branch(repository, branch)
+        except Exception as e:
+            logger.exception(e)
+            logger.error('Source repository could not be fetched. Check '
+                'that the address is correct and the branch exists.')
+            return
+    try:
+        dirlist = os.listdir(os.path.join(dst_folder, 'library'))
+    except OSError as e:
+        logger.error('The path provided ({0}) could not be found or didn\'t'
+            'contain a library/ folder.'.format(dst_folder))
+        return
+    for buildfile in dirlist:
         if buildfile == 'MAINTAINERS':
             continue
         f = open(os.path.join(dst_folder, 'library', buildfile))
+        linecnt = 0
         for line in f:
+            linecnt = linecnt + 1
             logger.debug('{0} ---> {1}'.format(buildfile, line))
             args = line.split()
             try:
@@ -58,30 +93,93 @@ def build_library(repository=None, branch=None, namespace=None, push=False):
                     else:
                         raise RuntimeError('Incorrect line format, '
                             'please refer to the docs')
-                img = build_repo(url, ref, buildfile, tag, namespace, push)
+                if prefill:
+                    logger.debug('Pulling {0} from official repository (cache '
+                        'fill)'.format(buildfile))
+                    client.pull(buildfile)
+                img = build_repo(url, ref, buildfile, tag, namespace, push,
+                    registry)
+                summary.add_success(buildfile, (linecnt, line), img)
                 processed['{0}@{1}'.format(url, ref)] = img
             except Exception as e:
                 logger.exception(e)
+                summary.add_exception(buildfile, (linecnt, line), e)
+
         f.close()
+    if dst_folder != repository:
+        rmtree(dst_folder, True)
+    for d in processed_folders:
+        rmtree(d, True)
+    summary.print_summary(logger)
 
 
-def build_repo(repository, ref, docker_repo, docker_tag, namespace, push):
+def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry):
     docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
     img_id = None
+    dst_folder = None
     if '{0}@{1}'.format(repository, ref) not in processed.keys():
         logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
-        dst_folder = git.clone(repository, ref)
+        if repository not in processed:
+            rep, dst_folder = git.clone(repository, ref)
+            processed[repository] = rep
+            processed_folders.append(dst_folder)
+        else:
+            dst_folder = git.checkout(processed[repository], ref)
         if not 'Dockerfile' in os.listdir(dst_folder):
             raise RuntimeError('Dockerfile not found in cloned repository')
         logger.info('Building using dockerfile...')
-        img_id, logs = client.build(path=dst_folder)
-
-    if not img_id:
+        img_id, logs = client.build(path=dst_folder, quiet=True)
+    else:
         img_id = processed['{0}@{1}'.format(repository, ref)]
     logger.info('Committing to {0}:{1}'.format(docker_repo,
         docker_tag or 'latest'))
     client.tag(img_id, docker_repo, docker_tag)
     if push:
-        logger.info('Pushing result to the main registry')
+        logger.info('Pushing result to registry {0}'.format(
+            registry or "default"))
+        if registry is not None:
+            docker_repo = '{0}/{1}'.format(registry, docker_repo)
+            logger.info('Also tagging {0}'.format(docker_repo))
+            client.tag(img_id, docker_repo, docker_tag)
         client.push(docker_repo)
     return img_id
+
+
+class Summary(object):
+    def __init__(self):
+        self._summary = {}
+        self._has_exc = False
+
+    def _add_data(self, image, linestr, data):
+        if image not in self._summary:
+            self._summary[image] = { linestr: data }
+        else:
+            self._summary[image][linestr] = data
+
+    def add_exception(self, image, line, exc):
+        lineno, linestr = line
+        self._add_data(image, linestr, { 'line': lineno, 'exc': str(exc) })
+        self._has_exc = True
+
+    def add_success(self, image, line, img_id):
+        lineno, linestr = line
+        self._add_data(image, linestr, { 'line': lineno, 'id': img_id })
+
+    def print_summary(self, logger=None):
+        linesep = ''.center(61, '-') + '\n'
+        s = 'BREW BUILD SUMMARY\n' + linesep
+        success = 'OVERALL SUCCESS: {}\n'.format(not self._has_exc)
+        details = linesep
+        for image, lines in self._summary.iteritems():
+            details = details + '{}\n{}'.format(image, linesep)
+            for linestr, data in lines.iteritems():
+                details = details + '{0:2} | {1} | {2:50}\n'.format(
+                    data['line'],
+                    'KO' if 'exc' in data else 'OK',
+                    data['exc'] if 'exc' in data else data['id']
+                )
+            details = details + linesep
+        if logger:
+            logger.info(s + success + details)
+        else:
+            print s, success, details

brew/git.py

@@ -16,6 +16,21 @@ def clone_tag(repo_url, tag, folder=None):
     return clone(repo_url, 'refs/tags/' + tag, folder)
 
 
+def checkout(rep, ref=None):
+    is_commit = False
+    if ref is None:
+        ref = 'refs/heads/master'
+    elif not ref.startswith('refs/'):
+        is_commit = True
+    if is_commit:
+        rep['HEAD'] = rep.commit(ref)
+    else:
+        rep['HEAD'] = rep.refs[ref]
+    indexfile = rep.index_path()
+    tree = rep["HEAD"].tree
+    index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
+    return rep.path
+
 def clone(repo_url, ref=None, folder=None):
     is_commit = False
     if ref is None:
@@ -45,4 +60,4 @@ def clone(repo_url, ref=None, folder=None):
     tree = rep["HEAD"].tree
     index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
     logger.debug("done")
-    return folder
+    return rep, folder

docker-brew

@@ -1,25 +1,35 @@
 #!/usr/bin/env python
 
 import argparse
+import sys
 
-import brew
-
-
-DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker'
-DEFAULT_BRANCH = 'library'
+try:
+    import brew
+except ImportError as e:
+    print str(e)
+    print 'Please install the required dependencies first'
+    print 'sudo pip install -r requirements.txt'
+    sys.exit(1)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser('Build the docker standard library')
     parser.add_argument('--push', action='store_true', default=False,
-        help='push generated repositories to the official registry')
+        help='Push generated repositories')
+    parser.add_argument('--debug', default=False, action='store_true',
+        help='Enable debugging output')
+    parser.add_argument('--noprefill', default=True, action='store_false',
+        dest='prefill', help='Disable cache prefill')
     parser.add_argument('-n', metavar='NAMESPACE', default='library',
-        help='namespace used for generated repositories.'
+        help='Namespace used for generated repositories.'
         ' Default is library')
-    parser.add_argument('-b', metavar='BRANCH', default=DEFAULT_BRANCH,
-        help='branch in the repository where the library definition'
-        ' files will be fetched. Default is ' + DEFAULT_BRANCH)
-    parser.add_argument('repository', default=DEFAULT_REPOSITORY, nargs='?',
-        help='git repository containing the library definition files.'
-        ' Default is ' + DEFAULT_REPOSITORY)
+    parser.add_argument('-b', metavar='BRANCH', default=brew.DEFAULT_BRANCH,
+        help='Branch in the repository where the library definition'
+        ' files will be fetched. Default is ' + brew.DEFAULT_BRANCH)
+    parser.add_argument('repository', default=brew.DEFAULT_REPOSITORY,
+        nargs='?', help='git repository containing the library definition'
+        ' files. Default is ' + brew.DEFAULT_REPOSITORY)
+    parser.add_argument('--reg', default=None, help='Registry address to'
+        ' push build results to. Also sets push to true.')
     args = parser.parse_args()
-    brew.build_library(args.repository, args.b, args.n, args.push)
+    brew.build_library(args.repository, args.b, args.n,
+        args.push or args.reg is not None, args.debug, args.prefill, args.reg)

requirements.txt

@@ -1,2 +1,2 @@
 dulwich==0.9.0
-docker==0.1.0
+-e git://github.com/dotcloud/docker-py.git#egg=docker-py