v2 to brew

stackbrew/app.py

@@ -3,12 +3,10 @@ import json
 
 import flask
 
-sys.path.append('./lib')
-
-import brew.v2 as brew
-import db
-import periodic
-import utils
+import brew
+import lib.db as db
+import lib.periodic as periodic
+import lib.utils as utils
 
 app = flask.Flask('stackbrew')
 config = None

stackbrew/brew-cli

@@ -55,9 +55,9 @@ if __name__ == '__main__':
     )
     args = parser.parse_args()
     if args.debug:
-        brew.v2.set_loglevel('DEBUG')
-    sb_library = brew.v2.StackbrewLibrary(args.repository, args.b)
-    builder = brew.v2.LocalBuilder(sb_library, args.n.split(','), args.targets)
+        brew.set_loglevel('DEBUG')
+    sb_library = brew.StackbrewLibrary(args.repository, args.b)
+    builder = brew.LocalBuilder(sb_library, args.n.split(','), args.targets)
     builder.build_repo_list()
     builder.build_all()
     if args.push:

stackbrew/brew/__init__.py

@@ -1,2 +1 @@
-from brew import build_library, DEFAULT_REPOSITORY, DEFAULT_BRANCH
-from . import v2
+from .brew import *

stackbrew/brew/brew.py

@@ -1,169 +1,255 @@
-import hashlib
+import json
 import logging
 import os
 import random
+import re
 from shutil import rmtree
 import string
 
 import docker
 
 import git
-from summary import Summary
 
-DEFAULT_REPOSITORY = 'git://github.com/shin-/brew'
+DEFAULT_REPOSITORY = 'git://github.com/dotcloud/stackbrew'
 DEFAULT_BRANCH = 'master'
 
-client = docker.Client(timeout=10000)
-processed = {}
-processed_folders = []
-
-
-def build_library(repository=None, branch=None, namespace=None, push=False,
-                  debug=False, prefill=True, registry=None, targetlist=None,
-                  repos_folder=None, logger=None):
-    ''' Entrypoint method build_library.
-        repository:     Repository containing a library/ folder. Can be a
-                        local path or git repository
-        branch:         If repository is a git repository, checkout this branch
-                        (default: DEFAULT_BRANCH)
-        namespace:      Created repositories will use the following namespace.
-                        (default: no namespace)
-        push:           If set to true, push images to the repository
-        debug:          Enables debug logging if set to True
-        prefill:        Retrieve images from public repository before building.
-                        Serves to prefill the builder cache.
-        registry:       URL to the private registry where results should be
-                        pushed. (only if push=True)
-        targetlist:     String indicating which library files are targeted by
-                        this build. Entries should be comma-separated. Default
-                        is all files.
-        repos_folder:   Fixed location where cloned repositories should be
-                        stored. Default is None, meaning folders are temporary
-                        and cleaned up after the build finishes.
-        logger:         Logger instance to use. Default is None, in which case
-                        build_library will create its own logger.
-    '''
-    dst_folder = None
-    summary = Summary()
-    if logger is None:
-        logger = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+                    level='INFO')
+
+
+def set_loglevel(level):
+    logger.setLevel(level)
+    git.logger.setLevel(level)
+
+
+class StackbrewError(Exception):
+    def __init__(self, message, cause=None):
+        super(StackbrewError, self).__init__(message)
+        self.cause = cause
+
+    def log(self, logger):
+        logger.exception(self)
+        if self.cause:
+            logger.error('The cause of this error is the following:')
+            logger.exception(self.cause)
+
+
+class StackbrewLibrary(object):
+    def __init__(self, repository, branch=None):
+        self.logger = logging.getLogger(__name__)
         logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                             level='INFO')
 
-    if repository is None:
-        repository = DEFAULT_REPOSITORY
-    if branch is None:
-        branch = DEFAULT_BRANCH
-    if debug:
-        logger.setLevel('DEBUG')
-    else:
-        logger.setLevel('INFO')
-    if targetlist is not None:
-        targetlist = targetlist.split(',')
-
-    if not repository.startswith(('https://', 'git://')):
-        logger.info('Repository provided assumed to be a local path')
-        dst_folder = repository
-
-    try:
-        client.version()
-    except Exception as e:
-        logger.error('Could not reach the docker daemon. Please make sure it '
-                     'is running.')
-        logger.warning('Also make sure you have access to the docker UNIX '
-                       'socket (use sudo)')
-        return
-
-    if not dst_folder:
-        logger.info('Cloning docker repo from {0}, branch: {1}'.format(
-            repository, branch))
+        self.branch = branch or DEFAULT_BRANCH
+        self.repository = repository
+        self.library = None
+        if not self.repository.startswith(('https://', 'git://')):
+            self.logger.info('Repository provided assumed to be a local path')
+            self.library = self.repository
+
+    def clone_library(self):
+        if self.library:
+            return self.library
+
         try:
-            rep, dst_folder = git.clone_branch(repository, branch)
-        except Exception as e:
-            logger.exception(e)
-            logger.error('Source repository could not be fetched. Check '
-                         'that the address is correct and the branch exists.')
-            return
-    try:
-        dirlist = os.listdir(os.path.join(dst_folder, 'library'))
-    except OSError as e:
-        logger.error('The path provided ({0}) could not be found or didn\'t'
-                     'contain a library/ folder.'.format(dst_folder))
-        return
-    for buildfile in dirlist:
-        if buildfile == 'MAINTAINERS':
-            continue
-        if (targetlist and buildfile not in targetlist):
-            continue
-        f = open(os.path.join(dst_folder, 'library', buildfile))
-        linecnt = 0
-        for line in f:
-            linecnt += 1
+            rep, library = git.clone_branch(self.repository, self.branch)
+            self.library = library
+        except git.GitException as e:
+            raise StackbrewError(
+                'Source repository could not be fetched. Ensure '
+                'the address is correct and the branch exists.',
+                e
+            )
+
+    def list_repositories(self):
+        if not self.library:
+            self.clone_library()
+        try:
+            return [e for e in os.listdir(
+                os.path.join(self.library, 'library')) if e != 'MAINTAINERS']
+        except OSError as e:
+            raise StackbrewError(
+                'The path provided ({0}) could not be found or '
+                'didn\'t contain a library/ folder'.format(self.library),
+                e
+            )
+
+
+class StackbrewRepo(object):
+    def __init__(self, name, definition_file):
+        self.buildlist = {}
+        self.git_folders = {}
+        self.name = name
+        for line in definition_file:
             if not line or line.strip() == '':
                 continue
             elif line.lstrip().startswith('#'):  # # It's a comment!
                 continue
-            logger.debug('{0} ---> {1}'.format(buildfile, line))
+            logger.debug(line)
+            tag, url, ref, dfile = self._parse_line(line)
+            if (url, ref, dfile) in self.buildlist:
+                self.buildlist[(url, ref, dfile)].append(tag)
+            else:
+                self.buildlist[(url, ref, dfile)] = [tag]
+
+    def _parse_line(self, line):
+        df_folder = '.'
+        args = line.split(':', 1)
+        if len(args) != 2:
+            logger.debug("Invalid line: {0}".format(line))
+            raise StackbrewError(
+                'Incorrect line format, please refer to the docs'
+            )
+
+        try:
+            repo = args[1].strip().split()
+            if len(repo) == 2:
+                df_folder = repo[1].strip()
+            url, ref = repo[0].strip().rsplit('@', 1)
+            return (args[0].strip(), url, ref, df_folder)
+        except ValueError:
+            logger.debug("Invalid line: {0}".format(line))
+            raise StackbrewError(
+                'Incorrect line format, please refer to the docs'
+            )
+
+    def list_versions(self):
+        return self.buildlist.keys()
+
+    def get_associated_tags(self, repo):
+        return self.buildlist.get(repo, None)
+
+    def add_git_repo(self, url, repo):
+        self.git_folders[url] = repo
+
+    def get_git_repo(self, url):
+        return self.git_folders.get(url, (None, None))
+
+
+class StackbrewBuilder(object):
+    def __init__(self, library, namespaces=None, targetlist=None,
+                 repo_cache=None):
+        self.lib = library
+        if not hasattr(self.lib, 'list_repositories'):
+            raise StackbrewError('Invalid library passed to StackbrewBuilder')
+        self.namespaces = namespaces or ['stackbrew']
+        self.targetlist = targetlist
+        self.repo_cache = repo_cache
+        self.history = {}
+
+    def build_repo_list(self):
+        self.repos = []
+        for repo in self.lib.list_repositories():
+            if self.targetlist and repo not in self.targetlist:
+                continue
             try:
-                tag, url, ref, dfile = parse_line(line, logger)
-                if prefill:
-                    logger.debug('Pulling {0} from official repository (cache '
-                                 'fill)'.format(buildfile))
-                    try:
-                        client.pull(buildfile)
-                    except:
-                        # Image is not on official repository, ignore prefill
-                        pass
-
-                img, commit = build_repo(url, ref, buildfile, dfile, tag,
-                                         namespace, push, registry,
-                                         repos_folder, logger)
-                summary.add_success(buildfile, (linecnt, line), img, commit)
-            except Exception as e:
-                logger.exception(e)
-                summary.add_exception(buildfile, (linecnt, line), e)
-
-        f.close()
-    cleanup(dst_folder, dst_folder != repository, repos_folder is None)
-    summary.print_summary(logger)
-    return summary
-
-
-def parse_line(line, logger):
-    df_folder = '.'
-    args = line.split(':', 1)
-    if len(args) != 2:
-        logger.debug("Invalid line: {0}".format(line))
-        raise RuntimeError('Incorrect line format, please refer to the docs')
-
-    try:
-        repo = args[1].strip().split()
-        if len(repo) == 2:
-            df_folder = repo[1].strip()
-        url, ref = repo[0].strip().rsplit('@', 1)
-        return (args[0].strip(), url, ref, df_folder)
-    except ValueError:
-        logger.debug("Invalid line: {0}".format(line))
-        raise RuntimeError('Incorrect line format, please refer to the docs')
-
-
-def cleanup(libfolder, clean_libfolder=False, clean_repos=True):
-    ''' Cleanup method called at the end of build_library.
-        libfolder:       Folder containing the library definition.
-        clean_libfolder: If set to True, libfolder will be removed.
-                         Only if libfolder was temporary
-        clean_repos: Remove library repos. Also resets module variables
-                     "processed" and "processed_folders" if set to true.
-    '''
-    global processed_folders
-    global processed
-    if clean_libfolder:
-        rmtree(libfolder, True)
-    if clean_repos:
-        for d in processed_folders:
-            rmtree(d, True)
-        processed_folders = []
-        processed = {}
+                with open(os.path.join(self.lib.library, 'library', repo)) as f:
+                    self.repos.append(StackbrewRepo(repo, f))
+            except IOError as e:
+                raise StackbrewError(
+                    'Failed to read definition file for {0}'.format(repo),
+                    e
+                )
+        for repo in self.repos:
+            for version in repo.list_versions():
+                logger.debug('{0}: {1}'.format(
+                    repo.name,
+                    ','.join(repo.get_associated_tags(version))
+                ))
+        return self.repos
+
+    def build_all(self, continue_on_error=True, callback=None):
+        self.pushlist = []
+        for repo in self.repos:
+            self.build_repo(repo, continue_on_error, callback)
+            for namespace in self.namespaces:
+                self.pushlist.append('/'.join([namespace, repo.name]))
+
+    def build_repo(self, repo, continue_on_error=True, callback=None):
+        for version in repo.list_versions():
+            try:
+                self.build_version(repo, version, callback)
+            except StackbrewError as e:
+                if not continue_on_error:
+                    raise e
+                e.log(logger)
+
+    def build_version(self, repo, version, callback=None):
+        if version in self.history:
+            return self.history[version], None
+        url, ref, dfile = version
+        try:
+            rep, dst_folder = self.clone_version(repo, version)
+        except StackbrewError as exc:
+            if callback:
+                callback(exc, repo, version, None, None)
+            raise exc
+        dockerfile_location = os.path.join(dst_folder, dfile)
+        if not 'Dockerfile' in os.listdir(dockerfile_location):
+            exc = StackbrewError('Dockerfile not found in cloned repository')
+            if callback:
+                callback(exc, repo, version, None, None)
+            raise exc
+        img_id, build_result = self.do_build(
+            repo, version, dockerfile_location, callback
+        )
+        self.history[version] = img_id
+        return img_id, build_result
+
+    def do_build(self, repo, version, dockerfile_location, callback=None):
+        raise NotImplementedError
+
+    def _clone_or_checkout(self, url, ref, dst_folder, rep):
+        if rep:
+            try:
+                # The ref already exists, we just need to checkout
+                dst_folder = git.checkout(rep, ref)
+            except git.GitException:
+                # ref is not present, try pulling it from the remote origin
+                rep, dst_folder = git.pull(url, rep, ref)
+            return rep, dst_folder
+
+        if dst_folder:
+            rmtree(dst_folder)
+        return git.clone(url, ref, dst_folder)
+
+    def clone_version(self, repo, version):
+        url, ref, dfile = version
+        rep, dst_folder = repo.get_git_repo(url)
+        if not dst_folder and self.repo_cache:
+            dst_folder = os.path.join(
+                self.repo_cache, repo.name + _random_suffix()
+            )
+            os.mkdir(dst_folder)
+        try:
+            rep, dst_folder = self._clone_or_checkout(
+                url, ref, dst_folder, rep
+            )
+        except Exception as e:
+            raise StackbrewError(
+                'Failed to clone repository {0}@{1}'.format(url, ref),
+                e
+            )
+
+        repo.add_git_repo(url, (rep, dst_folder))
+        return rep, dst_folder
+
+    def get_pushlist(self):
+        return self.pushlist
+
+    def push_all(self, continue_on_error=True, callback=None):
+        for repo in self.pushlist:
+            try:
+                self.do_push(repo, callback)
+            except StackbrewError as e:
+                if continue_on_error:
+                    e.log(logger)
+                else:
+                    raise e
+
+    def do_push(self, repo_name, callback=None):
+        raise NotImplementedError
 
 
 def _random_suffix():
@@ -172,117 +258,87 @@ def _random_suffix():
     ])
 
 
-def get_repo_hash(repo_url, ref, df_location):
-    h = hashlib.md5(repo_url)
-    h.update(ref)
-    h.update(df_location)
-    return h.hexdigest()
-
-
-def build_repo(repository, ref, docker_repo, dockerfile_location,
-               docker_tag, namespace, push, registry, repos_folder, logger):
-    ''' Builds one line of a library file.
-        repository:     URL of the git repository that needs to be built
-        ref:            Git reference (or commit ID) that needs to be built
-        docker_repo:    Name of the docker repository where the image will
-                        end up.
-        dockerfile_location: Folder containing the Dockerfile
-        docker_tag:     Tag for the image in the docker repository.
-        namespace:      Namespace for the docker repository.
-        push:           If the image should be pushed at the end of the build
-        registry:       URL to private registry where image should be pushed
-        repos_folder:   Directory where repositories should be cloned
-        logger:         Logger instance
-    '''
-    dst_folder = None
-    img_id = None
-    commit_id = None
-    repo_hash = get_repo_hash(repository, ref, dockerfile_location)
-    if repos_folder:
-        # Repositories are stored in a fixed location and can be reused
-        dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix())
-    docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
-
-    if repo_hash in processed.keys():
-        logger.info('[cache hit] {0}'.format(repo_hash))
-        logger.info('This ref has already been built, reusing image ID')
-        img_id = processed[repo_hash]
-        if ref.startswith('refs/'):
-            commit_id = processed[repository].ref(ref)
+class LocalBuilder(StackbrewBuilder):
+    def __init__(self, library, namespaces=None, targetlist=None,
+                 repo_cache=None):
+        super(LocalBuilder, self).__init__(
+            library, namespaces, targetlist, repo_cache
+        )
+        self.client = docker.Client(version='1.9', timeout=10000)
+        self.build_success_re = r'^Successfully built ([a-f0-9]+)\n$'
+
+    def do_build(self, repo, version, dockerfile_location, callback=None):
+        logger.info(
+            'Build start: {0} {1}'.format(repo.name, version)
+        )
+        build_result = self.client.build(path=dockerfile_location, rm=True,
+                                         stream=True, quiet=True)
+        img_id, logs = self._parse_result(build_result)
+        if not img_id:
+            exc = StackbrewError(
+                'Build failed for {0} ({1})'.format(repo.name, version)
+            )
+            if callback:
+                callback(exc, repo, version, None, logs)
+            raise exc
+        for tag in repo.get_associated_tags(version):
+            logger.info(
+                'Build success: {0} ({1}:{2})'.format(img_id, repo.name, tag)
+            )
+            for namespace in self.namespaces:
+                self.client.tag(img_id, '/'.join([namespace, repo.name]), tag)
+
+        if callback:
+            callback(None, repo, version, img_id, logs)
+        return img_id, build_result
+
+    def _parse_result(self, build_result):
+        if isinstance(build_result, tuple):
+            img_id, logs = build_result
+            return img_id, logs
         else:
-            commit_id = ref
-    else:
-        # Not already built
-        logger.info('[cache miss] {0}'.format(repo_hash))
-        rep = None
-        logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
-        if repository not in processed:  # Repository not cloned yet
+            lines = [line for line in build_result]
+            try:
+                parsed_lines = [json.loads(e).get('stream', '') for e in lines]
+            except ValueError:
+                # sometimes all the data is sent on a single line ????
+                #
+                # ValueError: Extra data: line 1 column 87 - line 1 column
+                # 33268 (char 86 - 33267)
+                line = lines[0]
+                # This ONLY works because every line is formatted as
+                # {"stream": STRING}
+                parsed_lines = [
+                    json.loads(obj).get('stream', '') for obj in
+                    re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line)
+                ]
+
+            for line in parsed_lines:
+                match = re.match(self.build_success_re, line)
+                if match:
+                    return match.group(1), parsed_lines
+            return None, parsed_lines
+
+    def do_push(self, repo_name, callback=None):
+        exc = None
+        for i in xrange(4):
             try:
-                rep, dst_folder = git.clone(repository, ref, dst_folder)
-            except Exception:
-                if dst_folder:
-                    rmtree(dst_folder)
-                ref = 'refs/tags/' + ref
-                rep, dst_folder = git.clone(repository, ref, dst_folder)
-            processed[repository] = rep
-            processed_folders.append(dst_folder)
+                pushlog = self.client.push(repo_name)
+                if '"error":"' in pushlog:
+                    raise RuntimeError(
+                        'Error while pushing: {0}'.format(pushlog)
+                    )
+                logger.info('Succesfully pushed {0}'.format(repo_name))
+            except Exception as e:
+                exc = e
+                continue
+            if callback:
+                callback(None, repo_name, pushlog)
+            return
+        if not callback:
+            raise StackbrewError(
+                'Error while pushing {0}'.format(repo_name),
+                exc
+            )
         else:
-            rep = processed[repository]
-            if ref in rep.refs:
-                # The ref already exists, we just need to checkout
-                dst_folder = git.checkout(rep, ref)
-            elif 'refs/tags/' + ref in rep.refs:
-                ref = 'refs/tags/' + ref
-                dst_folder = git.checkout(rep, ref)
-            else:  # ref is not present, try pulling it from the remote origin
-                try:
-                    rep, dst_folder = git.pull(repository, rep, ref)
-                except Exception:
-                    ref = 'refs/tags/' + ref
-                    rep, dst_folder = git.pull(repository, rep, ref)
-        dockerfile_location = os.path.join(dst_folder, dockerfile_location)
-        if not 'Dockerfile' in os.listdir(dockerfile_location):
-            raise RuntimeError('Dockerfile not found in cloned repository')
-        commit_id = rep.head()
-        logger.info('Building using dockerfile...')
-        img_id, logs = client.build(path=dockerfile_location, quiet=True)
-        if img_id is None:
-            logger.error('Image ID not found. Printing build logs...')
-            logger.debug(logs)
-            raise RuntimeError('Build failed')
-
-    logger.info('Committing to {0}:{1}'.format(docker_repo,
-                docker_tag or 'latest'))
-    client.tag(img_id, docker_repo, docker_tag)
-    logger.info("Registering as processed: {0}".format(repo_hash))
-    processed[repo_hash] = img_id
-    if push:
-        logger.info('Pushing result to registry {0}'.format(
-            registry or "default"))
-        push_repo(img_id, docker_repo, registry=registry, logger=logger)
-    return img_id, commit_id
-
-
-def push_repo(img_id, repo, registry=None, docker_tag=None, logger=None):
-    ''' Pushes a repository to a registry
-        img_id:     Image ID to push
-        repo:       Repository name where img_id should be tagged
-        registry:   Private registry where image needs to be pushed
-        docker_tag: Tag to be applied to the image in docker repo
-        logger:     Logger instance
-    '''
-    exc = None
-    if registry is not None:
-        repo = '{0}/{1}'.format(registry, repo)
-        logger.info('Also tagging {0}'.format(repo))
-        client.tag(img_id, repo, docker_tag)
-    for i in xrange(4):
-        try:
-            pushlog = client.push(repo)
-            if '"error":"' in pushlog:
-                raise RuntimeError('Error while pushing: {0}'.format(pushlog))
-        except Exception as e:
-            exc = e
-            continue
-        return
-    raise exc
+            return callback(exc, repo_name, None)

stackbrew/brew/v2.py

@@ -1,344 +0,0 @@
-import json
-import logging
-import os
-import random
-import re
-from shutil import rmtree
-import string
-
-import docker
-
-import git
-
-DEFAULT_REPOSITORY = 'git://github.com/dotcloud/stackbrew'
-DEFAULT_BRANCH = 'master'
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
-                    level='INFO')
-
-
-def set_loglevel(level):
-    logger.setLevel(level)
-    git.logger.setLevel(level)
-
-
-class StackbrewError(Exception):
-    def __init__(self, message, cause=None):
-        super(StackbrewError, self).__init__(message)
-        self.cause = cause
-
-    def log(self, logger):
-        logger.exception(self)
-        if self.cause:
-            logger.error('The cause of this error is the following:')
-            logger.exception(self.cause)
-
-
-class StackbrewLibrary(object):
-    def __init__(self, repository, branch=None):
-        self.logger = logging.getLogger(__name__)
-        logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
-                            level='INFO')
-
-        self.branch = branch or DEFAULT_BRANCH
-        self.repository = repository
-        self.library = None
-        if not self.repository.startswith(('https://', 'git://')):
-            self.logger.info('Repository provided assumed to be a local path')
-            self.library = self.repository
-
-    def clone_library(self):
-        if self.library:
-            return self.library
-
-        try:
-            rep, library = git.clone_branch(self.repository, self.branch)
-            self.library = library
-        except git.GitException as e:
-            raise StackbrewError(
-                'Source repository could not be fetched. Ensure '
-                'the address is correct and the branch exists.',
-                e
-            )
-
-    def list_repositories(self):
-        if not self.library:
-            self.clone_library()
-        try:
-            return [e for e in os.listdir(
-                os.path.join(self.library, 'library')) if e != 'MAINTAINERS']
-        except OSError as e:
-            raise StackbrewError(
-                'The path provided ({0}) could not be found or '
-                'didn\'t contain a library/ folder'.format(self.library),
-                e
-            )
-
-
-class StackbrewRepo(object):
-    def __init__(self, name, definition_file):
-        self.buildlist = {}
-        self.git_folders = {}
-        self.name = name
-        for line in definition_file:
-            if not line or line.strip() == '':
-                continue
-            elif line.lstrip().startswith('#'):  # # It's a comment!
-                continue
-            logger.debug(line)
-            tag, url, ref, dfile = self._parse_line(line)
-            if (url, ref, dfile) in self.buildlist:
-                self.buildlist[(url, ref, dfile)].append(tag)
-            else:
-                self.buildlist[(url, ref, dfile)] = [tag]
-
-    def _parse_line(self, line):
-        df_folder = '.'
-        args = line.split(':', 1)
-        if len(args) != 2:
-            logger.debug("Invalid line: {0}".format(line))
-            raise StackbrewError(
-                'Incorrect line format, please refer to the docs'
-            )
-
-        try:
-            repo = args[1].strip().split()
-            if len(repo) == 2:
-                df_folder = repo[1].strip()
-            url, ref = repo[0].strip().rsplit('@', 1)
-            return (args[0].strip(), url, ref, df_folder)
-        except ValueError:
-            logger.debug("Invalid line: {0}".format(line))
-            raise StackbrewError(
-                'Incorrect line format, please refer to the docs'
-            )
-
-    def list_versions(self):
-        return self.buildlist.keys()
-
-    def get_associated_tags(self, repo):
-        return self.buildlist.get(repo, None)
-
-    def add_git_repo(self, url, repo):
-        self.git_folders[url] = repo
-
-    def get_git_repo(self, url):
-        return self.git_folders.get(url, (None, None))
-
-
-class StackbrewBuilder(object):
-    def __init__(self, library, namespaces=None, targetlist=None,
-                 repo_cache=None):
-        self.lib = library
-        if not hasattr(self.lib, 'list_repositories'):
-            raise StackbrewError('Invalid library passed to StackbrewBuilder')
-        self.namespaces = namespaces or ['stackbrew']
-        self.targetlist = targetlist
-        self.repo_cache = repo_cache
-        self.history = {}
-
-    def build_repo_list(self):
-        self.repos = []
-        for repo in self.lib.list_repositories():
-            if self.targetlist and repo not in self.targetlist:
-                continue
-            try:
-                with open(os.path.join(self.lib.library, 'library', repo)) as f:
-                    self.repos.append(StackbrewRepo(repo, f))
-            except IOError as e:
-                raise StackbrewError(
-                    'Failed to read definition file for {0}'.format(repo),
-                    e
-                )
-        for repo in self.repos:
-            for version in repo.list_versions():
-                logger.debug('{0}: {1}'.format(
-                    repo.name,
-                    ','.join(repo.get_associated_tags(version))
-                ))
-        return self.repos
-
-    def build_all(self, continue_on_error=True, callback=None):
-        self.pushlist = []
-        for repo in self.repos:
-            self.build_repo(repo, continue_on_error, callback)
-            for namespace in self.namespaces:
-                self.pushlist.append('/'.join([namespace, repo.name]))
-
-    def build_repo(self, repo, continue_on_error=True, callback=None):
-        for version in repo.list_versions():
-            try:
-                self.build_version(repo, version, callback)
-            except StackbrewError as e:
-                if not continue_on_error:
-                    raise e
-                e.log(logger)
-
-    def build_version(self, repo, version, callback=None):
-        if version in self.history:
-            return self.history[version], None
-        url, ref, dfile = version
-        try:
-            rep, dst_folder = self.clone_version(repo, version)
-        except StackbrewError as exc:
-            if callback:
-                callback(exc, repo, version, None, None)
-            raise exc
-        dockerfile_location = os.path.join(dst_folder, dfile)
-        if not 'Dockerfile' in os.listdir(dockerfile_location):
-            exc = StackbrewError('Dockerfile not found in cloned repository')
-            if callback:
-                callback(exc, repo, version, None, None)
-            raise exc
-        img_id, build_result = self.do_build(
-            repo, version, dockerfile_location, callback
-        )
-        self.history[version] = img_id
-        return img_id, build_result
-
-    def do_build(self, repo, version, dockerfile_location, callback=None):
-        raise NotImplementedError
-
-    def _clone_or_checkout(self, url, ref, dst_folder, rep):
-        if rep:
-            try:
-                # The ref already exists, we just need to checkout
-                dst_folder = git.checkout(rep, ref)
-            except git.GitException:
-                # ref is not present, try pulling it from the remote origin
-                rep, dst_folder = git.pull(url, rep, ref)
-            return rep, dst_folder
-
-        if dst_folder:
-            rmtree(dst_folder)
-        return git.clone(url, ref, dst_folder)
-
-    def clone_version(self, repo, version):
-        url, ref, dfile = version
-        rep, dst_folder = repo.get_git_repo(url)
-        if not dst_folder and self.repo_cache:
-            dst_folder = os.path.join(
-                self.repo_cache, repo.name + _random_suffix()
-            )
-            os.mkdir(dst_folder)
-        try:
-            rep, dst_folder = self._clone_or_checkout(
-                url, ref, dst_folder, rep
-            )
-        except Exception as e:
-            raise StackbrewError(
-                'Failed to clone repository {0}@{1}'.format(url, ref),
-                e
-            )
-
-        repo.add_git_repo(url, (rep, dst_folder))
-        return rep, dst_folder
-
-    def get_pushlist(self):
-        return self.pushlist
-
-    def push_all(self, continue_on_error=True, callback=None):
-        for repo in self.pushlist:
-            try:
-                self.do_push(repo, callback)
-            except StackbrewError as e:
-                if continue_on_error:
-                    e.log(logger)
-                else:
-                    raise e
-
-    def do_push(self, repo_name, callback=None):
-        raise NotImplementedError
-
-
-def _random_suffix():
-    return ''.join([
-        random.choice(string.ascii_letters + string.digits) for i in xrange(6)
-    ])
-
-
-class LocalBuilder(StackbrewBuilder):
-    def __init__(self, library, namespaces=None, targetlist=None,
-                 repo_cache=None):
-        super(LocalBuilder, self).__init__(
-            library, namespaces, targetlist, repo_cache
-        )
-        self.client = docker.Client(version='1.9', timeout=10000)
-        self.build_success_re = r'^Successfully built ([a-f0-9]+)\n$'
-
-    def do_build(self, repo, version, dockerfile_location, callback=None):
-        logger.info(
-            'Build start: {0} {1}'.format(repo.name, version)
-        )
-        build_result = self.client.build(path=dockerfile_location, rm=True,
-                                         stream=True, quiet=True)
-        img_id, logs = self._parse_result(build_result)
-        if not img_id:
-            exc = StackbrewError(
-                'Build failed for {0} ({1})'.format(repo.name, version)
-            )
-            if callback:
-                callback(exc, repo, version, None, logs)
-            raise exc
-        for tag in repo.get_associated_tags(version):
-            logger.info(
-                'Build success: {0} ({1}:{2})'.format(img_id, repo.name, tag)
-            )
-            for namespace in self.namespaces:
-                self.client.tag(img_id, '/'.join([namespace, repo.name]), tag)
-
-        if callback:
-            callback(None, repo, version, img_id, logs)
-        return img_id, build_result
-
-    def _parse_result(self, build_result):
-        if isinstance(build_result, tuple):
-            img_id, logs = build_result
-            return img_id, logs
-        else:
-            lines = [line for line in build_result]
-            try:
-                parsed_lines = [json.loads(e).get('stream', '') for e in lines]
-            except ValueError:
-                # sometimes all the data is sent on a single line ????
-                #
-                # ValueError: Extra data: line 1 column 87 - line 1 column
-                # 33268 (char 86 - 33267)
-                line = lines[0]
-                # This ONLY works because every line is formatted as
-                # {"stream": STRING}
-                parsed_lines = [
-                    json.loads(obj).get('stream', '') for obj in
-                    re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line)
-                ]
-
-            for line in parsed_lines:
-                match = re.match(self.build_success_re, line)
-                if match:
-                    return match.group(1), parsed_lines
-            return None, parsed_lines
-
-    def do_push(self, repo_name, callback=None):
-        exc = None
-        for i in xrange(4):
-            try:
-                pushlog = self.client.push(repo_name)
-                if '"error":"' in pushlog:
-                    raise RuntimeError(
-                        'Error while pushing: {0}'.format(pushlog)
-                    )
-                logger.info('Succesfully pushed {0}'.format(repo_name))
-            except Exception as e:
-                exc = e
-                continue
-            if callback:
-                callback(None, repo_name, pushlog)
-            return
-        if not callback:
-            raise StackbrewError(
-                'Error while pushing {0}'.format(repo_name),
-                exc
-            )
-        else:
-            return callback(exc, repo_name, None)

stackbrew/create_db.py

@@ -1,8 +1,6 @@
 import sys
 
-sys.path.append('./lib')
-
-import db
+import lib.db as db
 
 data = db.DbManager(debug=True)
 data.generate_tables()