* pep8 refactoring in brew/brew.py

* added GIT commit_id to summary
* added config file in stackbrew (debug mode, enable pushes, build_interval)

brew/brew.py

@@ -16,13 +16,14 @@ processed_folders = []
 
 
 def build_library(repository=None, branch=None, namespace=None, push=False,
-        debug=False, prefill=True, registry=None, targetlist=None, logger=None):
+                  debug=False, prefill=True, registry=None, targetlist=None,
+                  logger=None):
     dst_folder = None
     summary = Summary()
     if logger is None:
         logger = logging.getLogger(__name__)
         logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
-            level='INFO')
+                            level='INFO')
 
     if repository is None:
         repository = DEFAULT_REPOSITORY
@@ -32,8 +33,6 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
         logger.setLevel('DEBUG')
     if targetlist is not None:
         targetlist = targetlist.split(',')
-    else:
-        targetlist = []
 
     if not (repository.startswith('https://') or repository.startswith('git://')):
         logger.info('Repository provided assumed to be a local path')
@@ -43,9 +42,9 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
         client.version()
     except Exception as e:
         logger.error('Could not reach the docker daemon. Please make sure it '
-            'is running.')
+                     'is running.')
         logger.warning('Also make sure you have access to the docker UNIX '
-            'socket (use sudo)')
+                       'socket (use sudo)')
         return
 
     #FIXME: set destination folder and only pull latest changes instead of
@@ -58,16 +57,16 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
         except Exception as e:
             logger.exception(e)
             logger.error('Source repository could not be fetched. Check '
-                'that the address is correct and the branch exists.')
+                         'that the address is correct and the branch exists.')
             return
     try:
         dirlist = os.listdir(os.path.join(dst_folder, 'library'))
     except OSError as e:
         logger.error('The path provided ({0}) could not be found or didn\'t'
-            'contain a library/ folder.'.format(dst_folder))
+                     'contain a library/ folder.'.format(dst_folder))
         return
     for buildfile in dirlist:
-        if buildfile == 'MAINTAINERS' or (len(targetlist) > 0 and buildfile not in targetlist):
+        if buildfile == 'MAINTAINERS' or (targetlist and buildfile not in targetlist):
             continue
         f = open(os.path.join(dst_folder, 'library', buildfile))
         linecnt = 0
@@ -78,7 +77,7 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
             try:
                 if len(args) > 3:
                     raise RuntimeError('Incorrect line format, '
-                        'please refer to the docs')
+                                       'please refer to the docs')
 
                 url = None
                 ref = 'refs/heads/master'
@@ -99,19 +98,19 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
                         ref = args[2][2:]
                     else:
                         raise RuntimeError('Incorrect line format, '
-                            'please refer to the docs')
+                                           'please refer to the docs')
                 if prefill:
                     logger.debug('Pulling {0} from official repository (cache '
-                        'fill)'.format(buildfile))
+                                 'fill)'.format(buildfile))
                     try:
                         client.pull('stackbrew/' + buildfile)
                     except:
                         # Image is not on official repository, ignore prefill
                         pass
 
-                img = build_repo(url, ref, buildfile, tag, namespace, push,
-                    registry, logger)
-                summary.add_success(buildfile, (linecnt, line), img)
+                img, commit = build_repo(url, ref, buildfile, tag, namespace,
+                                         push, registry, logger)
+                summary.add_success(buildfile, (linecnt, line), img, commit)
                 processed['{0}@{1}'.format(url, ref)] = img
             except Exception as e:
                 logger.exception(e)
@@ -127,34 +126,44 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
 
 
 def build_repo(repository, ref, docker_repo, docker_tag, namespace, push,
-    registry, logger):
+               registry, logger):
     docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo)
     img_id = None
+    commit_id = None
     dst_folder = None
     if '{0}@{1}'.format(repository, ref) not in processed.keys():
+        rep = None
         logger.info('Cloning {0} (ref: {1})'.format(repository, ref))
         if repository not in processed:
             rep, dst_folder = git.clone(repository, ref)
             processed[repository] = rep
             processed_folders.append(dst_folder)
         else:
-            dst_folder = git.checkout(processed[repository], ref)
+            rep = processed[repository]
+            dst_folder = git.checkout(rep, ref)
         if not 'Dockerfile' in os.listdir(dst_folder):
             raise RuntimeError('Dockerfile not found in cloned repository')
         logger.info('Building using dockerfile...')
         img_id, logs = client.build(path=dst_folder, quiet=True)
+        commit_id = rep.head()
     else:
+        logger.info('This ref has already been built, reusing image ID')
         img_id = processed['{0}@{1}'.format(repository, ref)]
+        if ref.startswith('refs/'):
+            commit_id = processed[repository].ref(ref)
+        else:
+            commit_id = ref
     logger.info('Committing to {0}:{1}'.format(docker_repo,
-        docker_tag or 'latest'))
+                docker_tag or 'latest'))
     client.tag(img_id, docker_repo, docker_tag)
     if push:
         logger.info('Pushing result to registry {0}'.format(
             registry or "default"))
-        push_repo(img_id, docker_repo, registry, logger)
-    return img_id
+        push_repo(img_id, docker_repo, registry=registry, logger=logger)
+    return img_id, commit_id
+
 
-def push_repo(img_id, repo, registry=None, logger):
+def push_repo(img_id, repo, registry=None, docker_tag=None, logger=None):
     exc = None
     if registry is not None:
         repo = '{0}/{1}'.format(registry, repo)
@@ -162,11 +171,11 @@ def push_repo(img_id, repo, registry=None, logger):
         client.tag(img_id, repo, docker_tag)
     for i in xrange(4):
         try:
-            pushlog = client.push(docker_repo)
+            pushlog = client.push(repo)
             if '"error":"' in pushlog:
                 raise RuntimeError('Error while pushing: {0}'.format(pushlog))
         except Exception as e:
             exc = e
             continue
         return
-    raise exc
+    raise exc

brew/summary.py

@@ -16,7 +16,8 @@ class Summary(object):
         self._has_exc = False
 
     def _add_data(self, image, linestr, data):
-        parts = linestr.split('\t')
+        linestr = linestr.strip('\n')
+        parts = linestr.split()
         tag = 'latest'
         source = None
         if len(parts) == 1:
@@ -27,7 +28,6 @@ class Summary(object):
         elif len(parts) == 3:
             tag = parts[0]
             source = '{}@{}'.format(parts[1], parts[2])
-        source = source.replace('\n', '')
         data.tag = tag
         data.source = source
         if image not in self._summary:
@@ -35,21 +35,23 @@ class Summary(object):
         else:
             self._summary[image][linestr] = data
 
-    def add_exception(self, image, line, exc):
+    def add_exception(self, image, line, exc, commit=None):
         lineno, linestr = line
         self._add_data(image, linestr, SummaryItem({
             'line': lineno,
             'exc': str(exc),
-            'repository': image
+            'repository': image,
+            'commit': commit
         }))
         self._has_exc = True
 
-    def add_success(self, image, line, img_id):
+    def add_success(self, image, line, img_id, commit=None):
         lineno, linestr = line
         self._add_data(image, linestr, SummaryItem({
             'line': lineno,
             'id': img_id,
-            'repository': image
+            'repository': image,
+            'commit': commit
         }))
 
     def print_summary(self, logger=None):

stackbrew/app.py

@@ -1,4 +1,5 @@
 import sys
+import json
 
 import flask
 
@@ -11,7 +12,10 @@ import periodic
 import utils
 
 app = flask.Flask('stackbrew')
-data = db.DbManager(debug=True)
+config = None
+with open('./config.json') as config_file:
+    config = json.load(config_file)
+data = db.DbManager(debug=config['debug'])
 
 
 @app.route('/')
@@ -39,18 +43,26 @@ def latest_success(repo_name):
     return utils.resp(app, result)
 
 
[email protected]('/build/force', method=['POST'])
+if config['debug']:
+    @app.route('/build/force', methods=['POST'])
+    def force_build():
+        build_task()
+
+
 def build_task():
     summary = brew.build_library(
         'https://github.com/shin-/brew.git', namespace='stackbrew',
-        debug=True, prefill=False, logger=app.logger
+        debug=config['debug'], push=config['push'], prefill=False,
+        logger=app.logger
     )
     data.insert_summary(summary)
 
 
 try:
-    periodic.init_task(build_task, 600, logger=app.logger)
+    periodic.init_task(build_task, config['build_interval'],
+                       logger=app.logger)
     app.logger.info('Periodic build task initiated.')
 except RuntimeError:
     app.logger.info('Periodic build task already locked.')
-app.run(debug=True)
+
+app.run(debug=config['debug'])

stackbrew/config.json

@@ -0,0 +1,5 @@
+{
+    "debug": true,
+    "push": false,
+    "build_interval": 6000
+}

stackbrew/lib/periodic.py

@@ -1,4 +1,4 @@
-#import atexit
+import atexit
 import os
 import threading
 
@@ -21,6 +21,9 @@ def init_task(fn, period, lockfile='/opt/stackbrew/brw.lock', logger=None):
 def clear_lockfiles(lockfiles):
     for lock in lockfiles:
         os.remove(lock)
-    lockfiles = []
 
-#atexit.register(clear_lockfiles, lockfiles)
+
+def on_exit(lockfiles):
+    clear_lockfiles(lockfiles)
+
+atexit.register(on_exit, lockfiles)