Merge pull request #63 from dotcloud/stackbrew-v2

Stackbrew v2

stackbrew/app.py

@@ -3,18 +3,19 @@ import json
 
 import flask
 
-sys.path.append('./lib')
-
 import brew
-import db
-import periodic
-import utils
+import lib.db as db
+import lib.periodic as periodic
+import lib.utils as utils
 
 app = flask.Flask('stackbrew')
 config = None
 with open('./config.json') as config_file:
     config = json.load(config_file)
 data = db.DbManager(config['db_url'], debug=config['debug'])
+history = {}
+brew.logger = app.logger
+brew.set_loglevel('DEBUG' if config['debug'] else 'INFO')
 
 
 @app.route('/')
@@ -46,15 +47,21 @@ if config['debug']:
     @app.route('/build/force', methods=['POST'])
     def force_build():
         build_task()
+        return utils.resp(app, 'OK')
 
 
 def build_task():
-    summary = brew.build_library(
-        config['library_repo'], namespace='stackbrew',
-        debug=config['debug'], push=config['push'], prefill=False,
-        repos_folder=config['repos_folder'], logger=app.logger
+    summary = data.new_summary(config['repos_folder'])
+    library = brew.StackbrewLibrary(config['library_repo'])
+    builder = brew.LocalBuilder(
+        library=library, namespaces=config['namespaces'],
+        repo_cache=config['repos_folder']
     )
-    data.insert_summary(summary)
+    builder.build_repo_list()
+    builder.history = history
+    builder.build_all(callback=summary.handle_build_result)
+    if config['push']:
+        builder.push_all()
 
 
 try:

stackbrew/brew-cli

@@ -27,10 +27,10 @@ if __name__ == '__main__':
         '--debug', default=False, action='store_true',
         help='Enable debugging output'
     )
-    parser.add_argument(
-        '--noprefill', default=True, action='store_false',
-        dest='prefill', help='Disable cache prefill'
-    )
+#    parser.add_argument(
+#        '--noprefill', default=True, action='store_false',
+#        dest='prefill', help='Disable cache prefill'
+#    )
     parser.add_argument(
         '-n', metavar='NAMESPACE', default='library',
         help='Namespace used for generated repositories. Default is library'
@@ -45,17 +45,25 @@ if __name__ == '__main__':
         nargs='?', help='git repository containing the library definition'
         ' files. Default is ' + brew.DEFAULT_REPOSITORY
     )
-    parser.add_argument(
-        '--reg', default=None, help='Registry address to'
-        ' push build results to. Also sets push to true.'
-    )
+#    parser.add_argument(
+#        '--reg', default=None, help='Registry address to'
+#        ' push build results to. Also sets push to true.'
+#    )
     parser.add_argument(
         '--targets', default=None, help='Comma-separated list'
         ' of images to build.'
     )
     args = parser.parse_args()
-    summary = brew.build_library(
-        args.repository, args.b, args.n, args.push or args.reg is not None,
-        args.debug, args.prefill, args.reg, args.targets, None, logger
-    )
-    sys.exit(summary.exit_code())
+    if args.debug:
+        brew.set_loglevel('DEBUG')
+    sb_library = brew.StackbrewLibrary(args.repository, args.b)
+    builder = brew.LocalBuilder(sb_library, args.n.split(','), args.targets)
+    builder.build_repo_list()
+    builder.build_all()
+    if args.push:
+        builder.push_all()
+
+#    summary = brew.build_library(
+#        args.repository, args.b, args.n, args.push or args.reg is not None,
+#        args.debug, args.prefill, args.reg, args.targets, None, logger
+#    )

stackbrew/brew/__init__.py

@@ -1 +1 @@
-from brew import build_library, DEFAULT_REPOSITORY, DEFAULT_BRANCH
+from .brew import *

stackbrew/brew/brew.py

@@ -1,167 +1,255 @@
-import hashlib
+import json
 import logging
 import os
 import random
+import re
 from shutil import rmtree
 import string
 
 import docker
 
 import git
-from summary import Summary
 
-DEFAULT_REPOSITORY = 'git://github.com/shin-/brew'
+DEFAULT_REPOSITORY = 'git://github.com/dotcloud/stackbrew'
 DEFAULT_BRANCH = 'master'
 
-client = docker.Client(timeout=10000)
-processed = {}
-processed_folders = []
-
-
-def build_library(repository=None, branch=None, namespace=None, push=False,
-                  debug=False, prefill=True, registry=None, targetlist=None,
-                  repos_folder=None, logger=None):
-    ''' Entrypoint method build_library.
-        repository:     Repository containing a library/ folder. Can be a
-                        local path or git repository
-        branch:         If repository is a git repository, checkout this branch
-                        (default: DEFAULT_BRANCH)
-        namespace:      Created repositories will use the following namespace.
-                        (default: no namespace)
-        push:           If set to true, push images to the repository
-        debug:          Enables debug logging if set to True
-        prefill:        Retrieve images from public repository before building.
-                        Serves to prefill the builder cache.
-        registry:       URL to the private registry where results should be
-                        pushed. (only if push=True)
-        targetlist:     String indicating which library files are targeted by
-                        this build. Entries should be comma-separated. Default
-                        is all files.
-        repos_folder:   Fixed location where cloned repositories should be
-                        stored. Default is None, meaning folders are temporary
-                        and cleaned up after the build finishes.
-        logger:         Logger instance to use. Default is None, in which case
-                        build_library will create its own logger.
-    '''
-    dst_folder = None
-    summary = Summary()
-    if logger is None:
-        logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+                    level='INFO')
+
+
+def set_loglevel(level):
+    logger.setLevel(level)
+    git.logger.setLevel(level)
+
+
+class StackbrewError(Exception):
+    def __init__(self, message, cause=None):
+        super(StackbrewError, self).__init__(message)
+        self.cause = cause
+
+    def log(self, logger):
+        logger.exception(self)
+        if self.cause:
+            logger.error('The cause of this error is the following:')
+            logger.exception(self.cause)
+
+
+class StackbrewLibrary(object):
+    def __init__(self, repository, branch=None):
+        self.logger = logging.getLogger(__name__)
         logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                             level='INFO')
 
-    if repository is None:
-        repository = DEFAULT_REPOSITORY
-    if branch is None:
-        branch = DEFAULT_BRANCH
-    if debug:
-        logger.setLevel('DEBUG')
-    if targetlist is not None:
-        targetlist = targetlist.split(',')
-
-    if not repository.startswith(('https://', 'git://')):
-        logger.info('Repository provided assumed to be a local path')
-        dst_folder = repository
-
-    try:
-        client.version()
-    except Exception as e:
-        logger.error('Could not reach the docker daemon. Please make sure it '
-                     'is running.')
-        logger.warning('Also make sure you have access to the docker UNIX '
-                       'socket (use sudo)')
-        return
-
-    if not dst_folder:
-        logger.info('Cloning docker repo from {0}, branch: {1}'.format(
-            repository, branch))
+        self.branch = branch or DEFAULT_BRANCH
+        self.repository = repository
+        self.library = None
+        if not self.repository.startswith(('https://', 'git://')):
+            self.logger.info('Repository provided assumed to be a local path')
+            self.library = self.repository
+
+    def clone_library(self):
+        if self.library:
+            return self.library
+
         try:
-            rep, dst_folder = git.clone_branch(repository, branch)
-        except Exception as e:
-            logger.exception(e)
-            logger.error('Source repository could not be fetched. Check '
-                         'that the address is correct and the branch exists.')
-            return
-    try:
-        dirlist = os.listdir(os.path.join(dst_folder, 'library'))
-    except OSError as e:
-        logger.error('The path provided ({0}) could not be found or didn\'t'
-                     'contain a library/ folder.'.format(dst_folder))
-        return
-    for buildfile in dirlist:
-        if buildfile == 'MAINTAINERS':
-            continue
-        if (targetlist and buildfile not in targetlist):
-            continue
-        f = open(os.path.join(dst_folder, 'library', buildfile))
-        linecnt = 0
-        for line in f:
-            linecnt += 1
+            rep, library = git.clone_branch(self.repository, self.branch)
+            self.library = library
+        except git.GitException as e:
+            raise StackbrewError(
+                'Source repository could not be fetched. Ensure '
+                'the address is correct and the branch exists.',
+                e
+            )
+
+    def list_repositories(self):
+        if not self.library:
+            self.clone_library()
+        try:
+            return [e for e in os.listdir(
+                os.path.join(self.library, 'library')) if e != 'MAINTAINERS']
+        except OSError as e:
+            raise StackbrewError(
+                'The path provided ({0}) could not be found or '
+                'didn\'t contain a library/ folder'.format(self.library),
+                e
+            )
+
+
+class StackbrewRepo(object):
+    def __init__(self, name, definition_file):
+        self.buildlist = {}
+        self.git_folders = {}
+        self.name = name
+        for line in definition_file:
             if not line or line.strip() == '':
                 continue
             elif line.lstrip().startswith('#'):  # # It's a comment!
                 continue
-            logger.debug('{0} ---> {1}'.format(buildfile, line))
+            logger.debug(line)
+            tag, url, ref, dfile = self._parse_line(line)
+            if (url, ref, dfile) in self.buildlist:
+                self.buildlist[(url, ref, dfile)].append(tag)
+            else:
+                self.buildlist[(url, ref, dfile)] = [tag]
+
+    def _parse_line(self, line):
+        df_folder = '.'
+        args = line.split(':', 1)
+        if len(args) != 2:
+            logger.debug("Invalid line: {0}".format(line))
+            raise StackbrewError(
+                'Incorrect line format, please refer to the docs'
+            )
+
+        try:
+            repo = args[1].strip().split()
+            if len(repo) == 2:
+                df_folder = repo[1].strip()
+            url, ref = repo[0].strip().rsplit('@', 1)
+            return (args[0].strip(), url, ref, df_folder)
+        except ValueError:
+            logger.debug("Invalid line: {0}".format(line))
+            raise StackbrewError(
+                'Incorrect line format, please refer to the docs'
+            )
+
+    def list_versions(self):
+        return self.buildlist.keys()
+
+    def get_associated_tags(self, repo):
+        return self.buildlist.get(repo, None)
+
+    def add_git_repo(self, url, repo):
+        self.git_folders[url] = repo
+
+    def get_git_repo(self, url):
+        return self.git_folders.get(url, (None, None))
+
+
+class StackbrewBuilder(object):
+    def __init__(self, library, namespaces=None, targetlist=None,
+                 repo_cache=None):
+        self.lib = library
+        if not hasattr(self.lib, 'list_repositories'):
+            raise StackbrewError('Invalid library passed to StackbrewBuilder')
+        self.namespaces = namespaces or ['stackbrew']
+        self.targetlist = targetlist
+        self.repo_cache = repo_cache
+        self.history = {}
+
+    def build_repo_list(self):
+        self.repos = []
+        for repo in self.lib.list_repositories():
+            if self.targetlist and repo not in self.targetlist:
+                continue
             try:
-                tag, url, ref, dfile = parse_line(line, logger)
-                if prefill:
-                    logger.debug('Pulling {0} from official repository (cache '
-                                 'fill)'.format(buildfile))
-                    try:
-                        client.pull(buildfile)
-                    except:
-                        # Image is not on official repository, ignore prefill
-                        pass
-
-                img, commit = build_repo(url, ref, buildfile, dfile, tag,
-                                         namespace, push, registry,
-                                         repos_folder, logger)
-                summary.add_success(buildfile, (linecnt, line), img, commit)
-            except Exception as e:
-                logger.exception(e)
-                summary.add_exception(buildfile, (linecnt, line), e)
-
-        f.close()
-    cleanup(dst_folder, dst_folder != repository, repos_folder is None)
-    summary.print_summary(logger)
-    return summary
-
-
-def parse_line(line, logger):
-    df_folder = '.'
-    args = line.split(':', 1)
-    if len(args) != 2:
-        logger.debug("Invalid line: {0}".format(line))
-        raise RuntimeError('Incorrect line format, please refer to the docs')
-
-    try:
-        repo = args[1].strip().split()
-        if len(repo) == 2:
-            df_folder = repo[1].strip()
-        url, ref = repo[0].strip().rsplit('@', 1)
-        return (args[0].strip(), url, ref, df_folder)
-    except ValueError:
-        logger.debug("Invalid line: {0}".format(line))
-        raise RuntimeError('Incorrect line format, please refer to the docs')
-
-
-def cleanup(libfolder, clean_libfolder=False, clean_repos=True):
-    ''' Cleanup method called at the end of build_library.
-        libfolder:       Folder containing the library definition.
-        clean_libfolder: If set to True, libfolder will be removed.
-                         Only if libfolder was temporary
-        clean_repos: Remove library repos. Also resets module variables
-                     "processed" and "processed_folders" if set to true.
-    '''
-    global processed_folders
-    global processed
-    if clean_libfolder:
-        rmtree(libfolder, True)
-    if clean_repos:
-        for d in processed_folders:
-            rmtree(d, True)
-        processed_folders = []
-        processed = {}
+                with open(os.path.join(self.lib.library, 'library', repo)) as f:
+                    self.repos.append(StackbrewRepo(repo, f))
+            except IOError as e:
+                raise StackbrewError(
+                    'Failed to read definition file for {0}'.format(repo),
+                    e
+                )
+        for repo in self.repos:
+            for version in repo.list_versions():
+                logger.debug('{0}: {1}'.format(
+                    repo.name,
+                    ','.join(repo.get_associated_tags(version))
+                ))
+        return self.repos
+
+    def build_all(self, continue_on_error=True, callback=None):
+        self.pushlist = []
+        for repo in self.repos:
+            self.build_repo(repo, continue_on_error, callback)
+            for namespace in self.namespaces:
+                self.pushlist.append('/'.join([namespace, repo.name]))
+
+    def build_repo(self, repo, continue_on_error=True, callback=None):
+        for version in repo.list_versions():
+            try:
+                self.build_version(repo, version, callback)
+            except StackbrewError as e:
+                if not continue_on_error:
+                    raise e
+                e.log(logger)
+
+    def build_version(self, repo, version, callback=None):
+        if version in self.history:
+            return self.history[version], None
+        url, ref, dfile = version
+        try:
+            rep, dst_folder = self.clone_version(repo, version)
+        except StackbrewError as exc:
+            if callback:
+                callback(exc, repo, version, None, None)
+            raise exc
+        dockerfile_location = os.path.join(dst_folder, dfile)
+        if not 'Dockerfile' in os.listdir(dockerfile_location):
+            exc = StackbrewError('Dockerfile not found in cloned repository')
+            if callback:
+                callback(exc, repo, version, None, None)
+            raise exc
+        img_id, build_result = self.do_build(
+            repo, version, dockerfile_location, callback
+        )
+        self.history[version] = img_id
+        return img_id, build_result
+
+    def do_build(self, repo, version, dockerfile_location, callback=None):
+        raise NotImplementedError
+
+    def _clone_or_checkout(self, url, ref, dst_folder, rep):
+        if rep:
+            try:
+                # The ref already exists, we just need to checkout
+                dst_folder = git.checkout(rep, ref)
+            except git.GitException:
+                # ref is not present, try pulling it from the remote origin
+                rep, dst_folder = git.pull(url, rep, ref)
+            return rep, dst_folder
+
+        if dst_folder:
+            rmtree(dst_folder)
+        return git.clone(url, ref, dst_folder)
+
+    def clone_version(self, repo, version):
+        url, ref, dfile = version
+        rep, dst_folder = repo.get_git_repo(url)
+        if not dst_folder and self.repo_cache:
+            dst_folder = os.path.join(
+                self.repo_cache, repo.name + _random_suffix()
+            )
+            os.mkdir(dst_folder)
+        try:
+            rep, dst_folder = self._clone_or_checkout(
+                url, ref, dst_folder, rep
+            )
+        except Exception as e:
+            raise StackbrewError(
+                'Failed to clone repository {0}@{1}'.format(url, ref),
+                e
+            )
+
+        repo.add_git_repo(url, (rep, dst_folder))
+        return rep, dst_folder
+
+    def get_pushlist(self):
+        return self.pushlist
+
+    def push_all(self, continue_on_error=True, callback=None):
+        for repo in self.pushlist:
+            try:
+                self.do_push(repo, callback)
+            except StackbrewError as e:
+                if continue_on_error:
+                    e.log(logger)
+                else:
+                    raise e
+
+    def do_push(self, repo_name, callback=None):
+        raise NotImplementedError
 
 
 def _random_suffix():
@@ -170,117 +258,87 @@ def _random_suffix():
     ])
 
 
-def get_repo_hash(repo_url, ref, df_location):
-    h = hashlib.md5(repo_url)
-    h.update(ref)
-    h.update(df_location)
-    return h.hexdigest()
-
-
-def build_repo(repository, ref, docker_repo, dockerfile_location,
-               docker_tag, namespace, push, registry, repos_folder, logger):
-    ''' Builds one line of a library file.
-        repository:     URL of the git repository that needs to be built
-        ref:            Git reference (or commit ID) that needs to be built
-        docker_repo:    Name of the docker repository where the image will
-                        end up.
-        dockerfile_location: Folder containing the Dockerfile
-        docker_tag:     Tag for the image in the docker repository.
-        namespace:      Namespace for the docker repository.
-        push:           If the image should be pushed at the end of the build
-        registry:       URL to private registry where image should be pushed
-        repos_folder:   Directory where repositories should be cloned
-        logger:         Logger instance
-    '''
-    dst_folder = None
-    img_id = None
-    commit_id = None
-    repo_hash = get_repo_hash(repository, ref, dockerfile_location)
-    if repos_folder:
-        # Repositories are stored in a fixed location and can be reused
-        dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix())
-    docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
-
-    if repo_hash in processed.keys():
-        logger.info('[cache hit] {0}'.format(repo_hash))
-        logger.info('This ref has already been built, reusing image ID')
-        img_id = processed[repo_hash]
-        if ref.startswith('refs/'):
-            commit_id = processed[repository].ref(ref)
+class LocalBuilder(StackbrewBuilder):
+    def __init__(self, library, namespaces=None, targetlist=None,
+                 repo_cache=None):
+        super(LocalBuilder, self).__init__(
+            library, namespaces, targetlist, repo_cache
+        )
+        self.client = docker.Client(version='1.9', timeout=10000)
+        self.build_success_re = r'^Successfully built ([a-f0-9]+)\n$'
+
+    def do_build(self, repo, version, dockerfile_location, callback=None):
+        logger.info(
+            'Build start: {0} {1}'.format(repo.name, version)
+        )
+        build_result = self.client.build(path=dockerfile_location, rm=True,
+                                         stream=True, quiet=True)
+        img_id, logs = self._parse_result(build_result)
+        if not img_id:
+            exc = StackbrewError(
+                'Build failed for {0} ({1})'.format(repo.name, version)
+            )
+            if callback:
+                callback(exc, repo, version, None, logs)
+            raise exc
+        for tag in repo.get_associated_tags(version):
+            logger.info(
+                'Build success: {0} ({1}:{2})'.format(img_id, repo.name, tag)
+            )
+            for namespace in self.namespaces:
+                self.client.tag(img_id, '/'.join([namespace, repo.name]), tag)
+
+        if callback:
+            callback(None, repo, version, img_id, logs)
+        return img_id, build_result
+
+    def _parse_result(self, build_result):
+        if isinstance(build_result, tuple):
+            img_id, logs = build_result
+            return img_id, logs
         else:
-            commit_id = ref
-    else:
-        # Not already built
-        logger.info('[cache miss] {0}'.format(repo_hash))
-        rep = None
-        logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
-        if repository not in processed:  # Repository not cloned yet
+            lines = [line for line in build_result]
+            try:
+                parsed_lines = [json.loads(e).get('stream', '') for e in lines]
+            except ValueError:
+                # sometimes all the data is sent on a single line ????
+                #
+                # ValueError: Extra data: line 1 column 87 - line 1 column
+                # 33268 (char 86 - 33267)
+                line = lines[0]
+                # This ONLY works because every line is formatted as
+                # {"stream": STRING}
+                parsed_lines = [
+                    json.loads(obj).get('stream', '') for obj in
+                    re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line)
+                ]
+
+            for line in parsed_lines:
+                match = re.match(self.build_success_re, line)
+                if match:
+                    return match.group(1), parsed_lines
+            return None, parsed_lines
+
+    def do_push(self, repo_name, callback=None):
+        exc = None
+        for i in xrange(4):
             try:
-                rep, dst_folder = git.clone(repository, ref, dst_folder)
-            except Exception:
-                if dst_folder:
-                    rmtree(dst_folder)
-                ref = 'refs/tags/' + ref
-                rep, dst_folder = git.clone(repository, ref, dst_folder)
-            processed[repository] = rep
-            processed_folders.append(dst_folder)
+                pushlog = self.client.push(repo_name)
+                if '"error":"' in pushlog:
+                    raise RuntimeError(
+                        'Error while pushing: {0}'.format(pushlog)
+                    )
+                logger.info('Succesfully pushed {0}'.format(repo_name))
+            except Exception as e:
+                exc = e
+                continue
+            if callback:
+                callback(None, repo_name, pushlog)
+            return
+        if not callback:
+            raise StackbrewError(
+                'Error while pushing {0}'.format(repo_name),
+                exc
+            )
         else:
-            rep = processed[repository]
-            if ref in rep.refs:
-                # The ref already exists, we just need to checkout
-                dst_folder = git.checkout(rep, ref)
-            elif 'refs/tags/' + ref in rep.refs:
-                ref = 'refs/tags/' + ref
-                dst_folder = git.checkout(rep, ref)
-            else:  # ref is not present, try pulling it from the remote origin
-                try:
-                    rep, dst_folder = git.pull(repository, rep, ref)
-                except Exception:
-                    ref = 'refs/tags/' + ref
-                    rep, dst_folder = git.pull(repository, rep, ref)
-        dockerfile_location = os.path.join(dst_folder, dockerfile_location)
-        if not 'Dockerfile' in os.listdir(dockerfile_location):
-            raise RuntimeError('Dockerfile not found in cloned repository')
-        commit_id = rep.head()
-        logger.info('Building using dockerfile...')
-        img_id, logs = client.build(path=dockerfile_location, quiet=True)
-        if img_id is None:
-            logger.error('Image ID not found. Printing build logs...')
-            logger.debug(logs)
-            raise RuntimeError('Build failed')
-
-    logger.info('Committing to {0}:{1}'.format(docker_repo,
-                docker_tag or 'latest'))
-    client.tag(img_id, docker_repo, docker_tag)
-    logger.info("Registering as processed: {0}".format(repo_hash))
-    processed[repo_hash] = img_id
-    if push:
-        logger.info('Pushing result to registry {0}'.format(
-            registry or "default"))
-        push_repo(img_id, docker_repo, registry=registry, logger=logger)
-    return img_id, commit_id
-
-
-def push_repo(img_id, repo, registry=None, docker_tag=None, logger=None):
-    ''' Pushes a repository to a registry
-        img_id:     Image ID to push
-        repo:       Repository name where img_id should be tagged
-        registry:   Private registry where image needs to be pushed
-        docker_tag: Tag to be applied to the image in docker repo
-        logger:     Logger instance
-    '''
-    exc = None
-    if registry is not None:
-        repo = '{0}/{1}'.format(registry, repo)
-        logger.info('Also tagging {0}'.format(repo))
-        client.tag(img_id, repo, docker_tag)
-    for i in xrange(4):
-        try:
-            pushlog = client.push(repo)
-            if '"error":"' in pushlog:
-                raise RuntimeError('Error while pushing: {0}'.format(pushlog))
-        except Exception as e:
-            exc = e
-            continue
-        return
-    raise exc
+            return callback(exc, repo_name, None)

stackbrew/brew/git.py

@@ -1,15 +1,34 @@
+import logging
 import os
+import subprocess
 import tempfile
-import logging
 
-from dulwich import index
-from dulwich.client import get_transport_and_path
-from dulwich.objects import Tag
-from dulwich.repo import Repo
 
 logger = logging.getLogger(__name__)
 
 
+class GitException(Exception):
+    pass
+
+
+class Repo(object):
+    def __init__(self, path, repo_url):
+        self.path = path
+        self.repo_url = repo_url
+
+    def clone(self):
+        logger.debug('Cloning {0} into {1}'.format(self.repo_url, self.path))
+        result = _execute('clone', [self.repo_url, '.'], self.path)
+        if result != 0:
+            raise GitException('git clone failed')
+
+
+def _execute(command, args, cwd):
+    cmd = ['git', command] + args
+    logger.debug('Executing "{0}" in {1}'.format(' '.join(cmd), cwd))
+    return subprocess.Popen(cmd, cwd=cwd).wait()
+
+
 def clone_branch(repo_url, branch="master", folder=None):
     return clone(repo_url, 'refs/heads/' + branch, folder)
 
@@ -21,53 +40,38 @@ def clone_tag(repo_url, tag, folder=None):
 def checkout(rep, ref=None):
     if ref is None:
         ref = 'refs/heads/master'
-    elif ref.startswith('refs/tags'):
-        ref = rep.ref(ref)
-    if isinstance(rep[ref], Tag):
-        rep['HEAD'] = rep[ref].object[1]
-    else:
-        rep['HEAD'] = rep.refs[ref]
-    indexfile = rep.index_path()
-    tree = rep["HEAD"].tree
-    index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
+    logger.debug("Checkout ref:{0} in {1}".format(ref, rep.path))
+    result = _execute('checkout', [ref], rep.path)
+
+    if result != 0:
+        raise GitException('git checkout failed')
+
     return rep.path
 
 
 def pull(origin, rep, ref=None):
-    clone(origin, ref, None, rep)
+    if ref is None:
+        ref = 'refs/heads/master'
+    logger.debug("Pull ref:{0} in {1}".format(ref, rep.path))
+    result = _execute('pull', ['origin', ref], rep.path)
+    if result != 0:
+        raise GitException('git pull failed')
+    checkout(rep, ref)
     return rep, rep.path
 
 
 def clone(repo_url, ref=None, folder=None, rep=None):
     if ref is None:
         ref = 'refs/heads/master'
-    logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref))
-    if not rep:
-        if folder is None:
-            folder = tempfile.mkdtemp()
-        else:
-            os.mkdir(folder)
-        logger.debug("folder = {0}".format(folder))
-        rep = Repo.init(folder)
-    client, relative_path = get_transport_and_path(repo_url)
-    logger.debug("client={0}".format(client))
-
-    remote_refs = client.fetch(relative_path, rep)
-    for k, v in remote_refs.iteritems():
-        try:
-            rep.refs.add_if_new(k, v)
-        except:
-            pass
-
-    if ref.startswith('refs/tags'):
-        ref = rep.ref(ref)
-
-    if isinstance(rep[ref], Tag):
-        rep['HEAD'] = rep[ref].object[1]
+    logger.debug("Cloning repo_url={0}, ref={1}".format(repo_url, ref))
+    if folder is None:
+        folder = tempfile.mkdtemp()
     else:
-        rep['HEAD'] = rep[ref]
-    indexfile = rep.index_path()
-    tree = rep["HEAD"].tree
-    index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
-    logger.debug("done")
+        os.mkdir(folder)
+    logger.debug("folder = {0}".format(folder))
+    rep = Repo(folder, repo_url)
+    rep.clone()
+    if ref:
+        checkout(rep, ref)
+
     return rep, folder

stackbrew/config.json

@@ -4,5 +4,7 @@
     "build_interval": 6000,
     "repos_folder": "/opt/stackbrew/repos",
     "db_url": "/opt/stackbrew/data.db",
-    "library_repo": "https://github.com/dotcloud/stackbrew.git"
+    "library_repo": "https://github.com/dotcloud/stackbrew.git",
+    "port": 2048,
+    "namespaces": ["sbv2"]
 }

stackbrew/create_db.py

@@ -1,8 +1,6 @@
 import sys
 
-sys.path.append('./lib')
-
-import db
+import lib.db as db
 
 data = db.DbManager(debug=True)
 data.generate_tables()

stackbrew/lib/db.py

@@ -24,6 +24,33 @@ summary_item = sql.Table(
 )
 
 
+class SummaryV2(object):
+    def __init__(self, engine, summary_id, errorlogs=None):
+        self.summary_id = summary_id
+        self._engine = engine
+        self.errorlogs = errorlogs
+
+    def handle_build_result(self, exc, repo, version, img_id, build_result):
+        c = self._engine.connect()
+        if exc and self.errorlogs:
+            if isinstance(build_result, list):
+                build_result = '\n'.join(build_result)
+            elif not isinstance(build_result, str):
+                build_result = str(build_result)
+            with open('{2}/{0}.{1}.error.log'.format(repo.name, version[1], self.errorlogs), 'w') as f:
+                f.write(build_result)
+        ins = summary_item.insert().values(
+            repo_name=repo.name,
+            exception=str(exc) if exc else None,
+            commit_id=version[1],
+            image_id=img_id,
+            source_desc=version[0],
+            tag=', '.join(repo.get_associated_tags(version)),
+            summary_id=self.summary_id
+        )
+        c.execute(ins)
+
+
 class DbManager(object):
     def __init__(self, db='/opt/stackbrew/data.db', debug=False):
         self._engine = sql.create_engine('sqlite:///' + db, echo=debug)
@@ -53,6 +80,15 @@ class DbManager(object):
                 c.execute(ins)
         return summary_id
 
+    def new_summary(self, errorlogs=None):
+        c = self._engine.connect()
+        ins = summary.insert().values(
+            result=True, build_date=str(datetime.datetime.now())
+        )
+        r = c.execute(ins)
+        summary_id = r.inserted_primary_key[0]
+        return SummaryV2(self._engine, summary_id, errorlogs)
+
     def latest_status(self):
         c = self._engine.connect()
         s = sql.select([summary]).order_by(summary.c.id.desc()).limit(1)

stackbrew/requirements.txt

@@ -1,4 +1,3 @@
 Flask==0.9
 SQLAlchemy==0.8.2
-dulwich==0.9.4
-docker-py==0.2.3
+docker-py==0.3.1